update
This commit is contained in:
parent
851daf6c48
commit
613583242c
178
glossary.ipynb
Normal file
178
glossary.ipynb
Normal file
@ -0,0 +1,178 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## IT Automation with Python\n",
|
||||
"\n",
|
||||
"## \n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## **Terms and definitions from Course 2**\n",
|
||||
"\n",
|
||||
"## A\n",
|
||||
"\n",
|
||||
"**Absolute path:** A full path to the resource in the file system\n",
|
||||
"\n",
|
||||
"**Alteration:** RegEx that matches any one of the alternatives separated by the pipe symbol\n",
|
||||
"\n",
|
||||
"**Automatic testing:** A process where software checks itself for errors and confirms that it works correctly\n",
|
||||
"\n",
|
||||
"## B\n",
|
||||
"\n",
|
||||
"**Backreference:** This is applied when using re.sub() to substitute the value of a capture group into the output\n",
|
||||
"\n",
|
||||
"**Bash:** The most commonly used shell on Linux\n",
|
||||
"\n",
|
||||
"**Bash script:** A script that contains multiple commands\n",
|
||||
"\n",
|
||||
"**Black-box tests:** A test where there is an awareness of what the program is supposed to do but not how it does it\n",
|
||||
"\n",
|
||||
"## C\n",
|
||||
"\n",
|
||||
"**Character classes:** These are written inside square brackets and let us list the characters we want to match inside of those brackets\n",
|
||||
"\n",
|
||||
"**Character ranges:** Ranges used to match a single character against a set of possibilities\n",
|
||||
"\n",
|
||||
"**Comma separated values (CSV):** A very common data format used to store data as segment of text separated by commas\n",
|
||||
"\n",
|
||||
"**Command line arguments:** Inputs provided to a program when running it from the command line\n",
|
||||
"\n",
|
||||
"**Compiled or compiled language:** Supportable code that can be executed on different platforms\n",
|
||||
"\n",
|
||||
"**Cut:** A command that can split and take only bits of each line using spaces\n",
|
||||
"\n",
|
||||
"## D\n",
|
||||
"\n",
|
||||
"**Dialects:** Rules that define how a CSV file is structured\n",
|
||||
"\n",
|
||||
"## E\n",
|
||||
"\n",
|
||||
"**Edge cases:** Inputs to code that produce unexpected results, found at the extreme ends of the ranges of input\n",
|
||||
"\n",
|
||||
"**Environment variables:** Settings and data stored outside a program that can be accessed by it to alter how the program behaves in a particular environment\n",
|
||||
"\n",
|
||||
"## F\n",
|
||||
"\n",
|
||||
"**File systems:** Methods and structures used to organize and control how data is stored and accessed\n",
|
||||
"\n",
|
||||
"## G\n",
|
||||
"\n",
|
||||
"**Globs:** Characters that create list of files, like the star and question mark\n",
|
||||
"\n",
|
||||
"**grep:** An especially easy to use yet extremely powerful tool for applying RegExes\n",
|
||||
"\n",
|
||||
"## I\n",
|
||||
"\n",
|
||||
"**Input / Output (I/O):** These streams are the basic mechanism for performing input and output operations in your programs\n",
|
||||
"\n",
|
||||
"**Integrated Development Environment (IDE):** A code editor with extra capabilities to simplify script writing\n",
|
||||
"\n",
|
||||
"**Interpreter or interpreted language:** An intermediary program used to execute the instructions specified in the code\n",
|
||||
"\n",
|
||||
"## K\n",
|
||||
"\n",
|
||||
"**Kernel:** The main core of an operating system that talks directly to hardware and manages the system’s resource\n",
|
||||
"\n",
|
||||
"## L\n",
|
||||
"\n",
|
||||
"**Linux:** An open source operating system where the software is free to share, modify, and distribute\n",
|
||||
"\n",
|
||||
"**Log files:** Log files are records or text files that store a history of events, actions, or errors generated by a computer system, software, or application for diagnostic, troubleshooting, or auditing purposes\n",
|
||||
"\n",
|
||||
"**Lookahead:** RegEx that matches a pattern only if it’s followed by another pattern\n",
|
||||
"\n",
|
||||
"## M\n",
|
||||
"\n",
|
||||
"**Mac OS:** Operating system developed by Apple\n",
|
||||
"\n",
|
||||
"**Mode:** The format controlling what you can do with a recently opened file\n",
|
||||
"\n",
|
||||
"## O\n",
|
||||
"\n",
|
||||
"**Operating system (OS):** Software that manages everything that goes on in the computer, composed of two main parts: the kernel and the user space\n",
|
||||
"\n",
|
||||
"## P\n",
|
||||
"\n",
|
||||
"**Pipes:** A process of connecting the output of one program to the input of another\n",
|
||||
"\n",
|
||||
"**Piping:** A process of connecting multiple scripts, commands, or other programs together into a data processing pipeline\n",
|
||||
"\n",
|
||||
"**Pytest:** A powerful Python testing tool that assists programmers in writing more effective and stable programs\n",
|
||||
"\n",
|
||||
"## Q\n",
|
||||
"\n",
|
||||
"**Qwiklabs:** An online learning environment or virtual machine to simulate real-world scenarios\n",
|
||||
"\n",
|
||||
"## R\n",
|
||||
"\n",
|
||||
"**Reader objects:** Object that represents an element or entity within a scene that needs to be rendered to the screen\n",
|
||||
"\n",
|
||||
"**Redirection:** A process of sending a stream to a different destination\n",
|
||||
"\n",
|
||||
"**Regular expression:** A search query for text that's expressed by string pattern, also known as RegEx or RegExp\n",
|
||||
"\n",
|
||||
"**Relative path:** A portion of a path to show where the resource is located in relation to the current working directory\n",
|
||||
"\n",
|
||||
"## S\n",
|
||||
"\n",
|
||||
"**Signals:** Tokens delivered to running processes to indicate a desired action\n",
|
||||
"\n",
|
||||
"**Software testing:** A process of evaluating computer code to determine whether or not it does what is expected\n",
|
||||
"\n",
|
||||
"**Standard input stream commonly (STDIN):** A channel between a program and a source of input\n",
|
||||
"\n",
|
||||
"**Standard output stream (STDOUT):** A pathway between a program and a target of output, like a display\n",
|
||||
"\n",
|
||||
"**Standard error (STDERR):** This displays output like standard out, but is used specifically as a channel to show error messages and diagnostics from the program\n",
|
||||
"\n",
|
||||
"**Shell:** The application that reads and executes all commands\n",
|
||||
"\n",
|
||||
"**Subprocesses:** A process to call and run other applications from within Python, including other Python scripts\n",
|
||||
"\n",
|
||||
"## T\n",
|
||||
"\n",
|
||||
"**Test case:** This is the individual unit of testing that looks for a specific response to a set of inputs\n",
|
||||
"\n",
|
||||
"**Test fixture:** This prepared to perform one or more tests\n",
|
||||
"\n",
|
||||
"**Test suite:** This is used to compile tests that should be executed together\n",
|
||||
"\n",
|
||||
"**Test runner:** This runs the test and provides developers with the outcome’s data\n",
|
||||
"\n",
|
||||
"## U\n",
|
||||
"\n",
|
||||
"**unittest:** A set of Python tools to construct and run unit tests\n",
|
||||
"\n",
|
||||
"**Unit tests:** A test to verify that small isolated parts of a program work correctly\n",
|
||||
"\n",
|
||||
"**User space:** Everything outside of the kernel that users interact with directly\n",
|
||||
"\n",
|
||||
"## V\n",
|
||||
"\n",
|
||||
"**Virtual environment:** A tool that allows you to create isolated environments for your Python projects\n",
|
||||
"\n",
|
||||
"**Virtual machine (VM):** A computer simulated through software\n",
|
||||
"\n",
|
||||
"## W\n",
|
||||
"\n",
|
||||
"**White-box test:** A test where test creator knows how the code works and can write test cases that use the understanding to make sure it performs as expected\n",
|
||||
"\n",
|
||||
"**Wildcard:** A character that can match more than one character\n",
|
||||
"\n",
|
||||
"**Windows OS:** Operating system developed by Microsoft\n",
|
||||
"\n",
|
||||
"**Writer objects:** The capability to write data to a CSV file"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
120
module7.ipynb
120
module7.ipynb
@ -7,12 +7,130 @@
|
||||
"# Final Project"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Log analysis using regular expressions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
"source": [
|
||||
"grep \"ERROR\" syslog.log\n",
|
||||
"grep \"ERROR Tried to add information to closed ticket\" syslog.log"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import re\n",
|
||||
"line = \"May 27 11:45:40 ubuntu.local ticky: INFO: Created ticket [#1234] (username)\"\n",
|
||||
"re.search(r\"ticky: INFO: ([\\w ]*) \", line)\n",
|
||||
"# <re.Match object; span=(29, 56), match='ticky: INFO Created ticket '>\n",
|
||||
"line = \"May 27 11:45:40 ubuntu.local ticky: ERROR Error creating ticket [#1234] (username)\"\n",
|
||||
"re.search(r\"ticky: ERROR ([\\w ]*) \", line)\n",
|
||||
"# <re.Match object; span=(29, 64), match='ticky: ERROR Error creating ticket '>\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"fruit = {\"oranges\": 3, \"apples\": 5, \"bananas\": 7, \"pears\": 2}\n",
|
||||
"sorted(fruit.items())\n",
|
||||
"# [('apples', 5), ('bananas', 7), ('oranges', 3), ('pears', 2)]\n",
|
||||
"\n",
|
||||
"import operator\n",
|
||||
"sorted(fruit.items(), key=operator.itemgetter(0))\n",
|
||||
"# [('apples', 5), ('bananas', 7), ('oranges', 3), ('pears', 2)]\n",
|
||||
"\n",
|
||||
"sorted(fruit.items(), key=operator.itemgetter(1))\n",
|
||||
"# [('pears', 2), ('oranges', 3), ('apples', 5), ('bananas', 7)]\n",
|
||||
"\n",
|
||||
"sorted(fruit.items(), key = operator.itemgetter(1), reverse=True)\n",
|
||||
"# [('bananas', 7), ('apples', 5), ('oranges', 3), ('pears', 2)]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#!/usr/bin/env python3\n",
|
||||
"import sys\n",
|
||||
"import re\n",
|
||||
"import operator\n",
|
||||
"import csv\n",
|
||||
"\n",
|
||||
"# Dict: Count number of entries for each user\n",
|
||||
"per_user = {} # Splitting between INFO and ERROR\n",
|
||||
"# Dict: Number of different error messages\n",
|
||||
"errors = {}\n",
|
||||
"\n",
|
||||
"# * Read file and create dictionaries\n",
|
||||
"with open('syslog.log') as file:\n",
|
||||
" # read each line\n",
|
||||
" for line in file.readlines():\n",
|
||||
" # regex search\n",
|
||||
" # * Sample Line of log file\n",
|
||||
" # \"May 27 11:45:40 ubuntu.local ticky: INFO: Created ticket [#1234] (username)\"\n",
|
||||
" match = re.search(\n",
|
||||
" r\"ticky: ([\\w+]*):? ([\\w' ]*)[\\[[#0-9]*\\]?]? ?\\((.*)\\)$\", line)\n",
|
||||
" code, error_msg, user = match.group(1), match.group(2), match.group(3)\n",
|
||||
"\n",
|
||||
" # Populates error dict with ERROR messages from log file\n",
|
||||
" if error_msg not in errors.keys():\n",
|
||||
" errors[error_msg] = 1\n",
|
||||
" else:\n",
|
||||
" errors[error_msg] += 1\n",
|
||||
" # Populates per_user dict with users and default values\n",
|
||||
" if user not in per_user.keys():\n",
|
||||
" per_user[user] = {}\n",
|
||||
" per_user[user]['INFO'] = 0\n",
|
||||
" per_user[user]['ERROR'] = 0\n",
|
||||
" # Populates per_user dict with users logs entry\n",
|
||||
" if code == 'INFO':\n",
|
||||
" if user not in per_user.keys():\n",
|
||||
" per_user[user] = {}\n",
|
||||
" per_user[user]['INFO'] = 0\n",
|
||||
" else:\n",
|
||||
" per_user[user][\"INFO\"] += 1\n",
|
||||
" elif code == 'ERROR':\n",
|
||||
" if user not in per_user.keys():\n",
|
||||
" per_user[user] = {}\n",
|
||||
" per_user[user]['INFO'] = 0\n",
|
||||
" else:\n",
|
||||
" per_user[user]['ERROR'] += 1\n",
|
||||
"\n",
|
||||
"# Sorted by VALUE (Most common to least common)\n",
|
||||
"errors_list = sorted(errors.items(), key=operator.itemgetter(1), reverse=True)\n",
|
||||
"# Sorted by USERNAME\n",
|
||||
"per_user_list = sorted(per_user.items(), key=operator.itemgetter(0))\n",
|
||||
"\n",
|
||||
"file.close()\n",
|
||||
"# Insert at the beginning of the list\n",
|
||||
"errors_list.insert(0, ('Error', 'Count'))\n",
|
||||
"# * Create CSV file user_statistics\n",
|
||||
"with open('user_statistics.csv', 'w', newline='') as user_csv:\n",
|
||||
" for key, value in per_user_list:\n",
|
||||
" user_csv.write(str(key) + ',' +\n",
|
||||
" str(value['INFO']) + ',' + str(value['ERROR'])+'\\n')\n",
|
||||
"# * Create CSV error_message\n",
|
||||
"with open('error_message.csv', 'w', newline='') as error_csv:\n",
|
||||
" for key, value in errors_list:\n",
|
||||
" error_csv.write(str(key) + ' ' + str(value))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
217
takeaways.ipynb
Normal file
217
takeaways.ipynb
Normal file
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user