diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..09e8e0c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,17 @@ +name: Validate Shell Script + +on: [push] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Validate Shell Script + run: | + # Install shellcheck + sudo apt-get update + sudo apt-get install -y shellcheck + # Run shellcheck on the script + shellcheck environment_setup.sh + shellcheck performance-monitoring.sh diff --git a/apache_log_parsing/apache_log_parsing_v3.py b/apache_log_parsing/apache_log_parsing_v3.py new file mode 100644 index 0000000..9246e99 --- /dev/null +++ b/apache_log_parsing/apache_log_parsing_v3.py @@ -0,0 +1,48 @@ +import re +import csv +import argparse +from collections import Counter + +IP_REGEX = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" + +# Create the parser +parser = argparse.ArgumentParser(description='Reading the log and CSV file') +parser.add_argument("--l", "--logfile", + help='Please enter the logfile to parse', + dest="logfile", + type=argparse.FileType('r'), + required=True) + +def extract_ips(logfile): + """Extracts all IP addresses from the given logfile.""" + try: + with open(logfile, 'r') as f: + log = f.read() + except Exception as e: + print(f"Error reading file: {e}") + return [] + return re.findall(IP_REGEX, log) + +def count_ips(ip_list): + """Count the occurrence of each IP address in the list.""" + return Counter(ip_list) + +def write_csv(counter): + """Write the counter data to a CSV file.""" + try: + with open("ip_count.csv", 'w') as f: + writer = csv.DictWriter(f, fieldnames=["IP_Address", "Count"]) + writer.writeheader() + for item, count in counter.items(): + writer.writerow({"IP_Address": item, "Count": count}) + except Exception as e: + print(f"Error writing CSV file: {e}") + +def main(): + args = parser.parse_args() + ip_list = extract_ips(args.logfile) + ip_counter = count_ips(ip_list) + write_csv(ip_counter) + +if __name__ == '__main__': + main() diff --git a/apache_log_parsing/apache_log_parsing_v4.py b/apache_log_parsing/apache_log_parsing_v4.py new file mode 100644 index 0000000..211521f --- /dev/null +++ b/apache_log_parsing/apache_log_parsing_v4.py @@ -0,0 +1,45 @@ +import re +import csv +import argparse +from collections import Counter + +IP_REGEX = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" + +# Create the parser +parser = argparse.ArgumentParser(description='Parse a log file and output IP address occurrences to a CSV file.') +parser.add_argument("--l", "--logfile", + help='Logfile to parse', + dest="logfile", + type=argparse.FileType('r'), + required=True) +parser.add_argument("--o", "--output", + help='Output CSV file name', + dest="outputfile", + type=str, + default="ip_count.csv") + +def extract_ips(logfile): + """Extracts all IP addresses from the given logfile.""" + return re.findall(IP_REGEX, logfile.read()) + +def count_ips(ip_list): + """Count the occurrence of each IP address in the list.""" + return Counter(ip_list) + +def write_csv(counter, filename): + """Write the counter data to a CSV file.""" + with open(filename, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=["IP_Address", "Count"]) + writer.writeheader() + for item, count in counter.items(): + writer.writerow({"IP_Address": item, "Count": count}) + print(f"IP counts written to {filename}") + +def main(): + args = parser.parse_args() + ip_list = extract_ips(args.logfile) + ip_counter = count_ips(ip_list) + write_csv(ip_counter, args.outputfile) + +if __name__ == '__main__': + main() diff --git a/aws_resource_notification/aws_billing_per_resource.sh b/aws_resource_notification/aws_billing_per_resource.sh new file mode 100644 index 0000000..77db5de --- /dev/null +++ b/aws_resource_notification/aws_billing_per_resource.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Set the start and end dates for the time period of cost report +start_date=$(date +%Y-%m-01) +end_date=$(date -d "$(date +%Y-%m-01) +1 month -1 day" +%Y-%m-%d) + +# Get the cost and usage report by resource,the level of granularity (e.g., hourly, daily, or monthly). Check this doc for more info https://docs.aws.amazon.com/cli/latest/reference/ce/get-cost-and-usage.html +usage_report=$(aws ce get-cost-and-usage --time-period Start=$start_date,End=$end_date --granularity MONTHLY --metrics "BlendedCost" "UnblendedCost" "UsageQuantity" --group-by Type=DIMENSION,Key=SERVICE --query 'ResultsByTime[0].Groups') + +# Verify if the cost and usage report retrieval was successful +if [ -z "$usage_report" ]; then + echo "Error: To retrieve cost and usage information by resource." + exit 1 +fi + +# Format the usage and cost report by resource as a table +report_table=$(echo "$usage_report" | jq -r '.[] | [.Keys[0], .Metrics.BlendedCost.Amount, .Metrics.UnblendedCost.Amount, .Metrics.UsageQuantity.Amount] | @tsv' | column -t -s $'\t' | awk 'BEGIN { print "Resource\tBlendedCost\tUnblendedCost\tUsageQuantity" } { print }') + +# Check if the EC2 or S3 resource usage exceeds $5 +if echo "$report_table" | grep -E '(^AmazonEC2|^AmazonS3)' | awk -F '\t' '$2 > 5 || $3 > 5'; then + # Send an email with the cost and usage report by resource(Don't forget to verify the email address: https://docs.aws.amazon.com/cli/latest/reference/ses/send-email.html) + aws ses send-email --from "sender@example.com" --to "recipient@example.com" --subject "AWS cost and usage report for the month" --text "$(echo "$report_table")" +fi diff --git a/boto3/cleanup_old_ami/cleanup_old_ami_v2.py b/boto3/cleanup_old_ami/cleanup_old_ami_v2.py new file mode 100644 index 0000000..10e5c8f --- /dev/null +++ b/boto3/cleanup_old_ami/cleanup_old_ami_v2.py @@ -0,0 +1,23 @@ +import boto3 +from operator import itemgetter + +# Define the maximum number of AMIs to keep +max_count = 5 + +# Connect to AWS using the default profile +ec2 = boto3.client('ec2') + +# Retrieve a list of all AMIs +response = ec2.describe_images(Owners=['self']) + +# Sort the list of AMIs by creation date in descending order +sorted_images = sorted(response['Images'], key=itemgetter('CreationDate'), reverse=True) + +# Loop through the list of AMIs and delete any that are beyond the max_count threshold +for index, image in enumerate(sorted_images): + if index >= max_count: + ec2.deregister_image(ImageId=image['ImageId']) + print(f"Deregistered AMI {image['ImageId']}") + else: + print(f"Keeping AMI {image['ImageId']}") + break diff --git a/boto3/cleanup_old_ami/cleanup_old_ami_v3.py b/boto3/cleanup_old_ami/cleanup_old_ami_v3.py new file mode 100644 index 0000000..471dbac --- /dev/null +++ b/boto3/cleanup_old_ami/cleanup_old_ami_v3.py @@ -0,0 +1,34 @@ +import boto3 +from datetime import datetime +from dateutil.parser import parse + +# Set the tag key and value to use for filtering AMIs +tag_key = "my-tag-key" +tag_value = "my-tag-value" + +# Set the maximum number of days to keep an AMI +max_age_days = 2 + +# Connect to EC2 using the default profile +client = boto3.client("ec2") + +# Get a list of all the AMIs with the specified tag +my_ami = client.describe_images( + Owners=["self"], + Filters=[ + { + "Name": f"tag:{tag_key}", + "Values": [tag_value] + } + ] +)["Images"] + +# Loop through the list of AMIs and delete any that are older than the maximum age +for ami in my_ami: + creation_date = parse(ami["CreationDate"]).replace(tzinfo=None) + ami_id = ami["ImageId"] + age_in_days = (datetime.now() - creation_date).days + print(f"Checking AMI {ami_id} with age {age_in_days} days") + if age_in_days > max_age_days: + client.deregister_image(ImageId=ami_id) + print(f"Deleted AMI {ami_id} with age {age_in_days} days") diff --git a/boto3/cleanup_old_ami/cleanup_old_ami_v4.py b/boto3/cleanup_old_ami/cleanup_old_ami_v4.py new file mode 100644 index 0000000..4fdc493 --- /dev/null +++ b/boto3/cleanup_old_ami/cleanup_old_ami_v4.py @@ -0,0 +1,39 @@ +import boto3 +from datetime import datetime +from dateutil.parser import parse + +# Set the tag key and value to use for filtering AMIs +tag_key = "my-tag-key" +tag_value = "my-tag-value" + +# Set the maximum number of days to keep an AMI +max_age_days = 2 + +# Set the list of regions to clean up AMIs +regions = ["us-west-1", "us-west-2", "us-east-1"] + +# Loop through each region and clean up AMIs +for region in regions: + # Connect to EC2 in the current region using the default profile + client = boto3.client("ec2", region_name=region) + + # Get a list of all the AMIs with the specified tag + my_ami = client.describe_images( + Owners=["self"], + Filters=[ + { + "Name": f"tag:{tag_key}", + "Values": [tag_value] + } + ] + )["Images"] + + # Loop through the list of AMIs and delete any that are older than the maximum age + for ami in my_ami: + creation_date = parse(ami["CreationDate"]).replace(tzinfo=None) + ami_id = ami["ImageId"] + age_in_days = (datetime.now() - creation_date).days + print(f"Checking AMI {ami_id} in region {region} with age {age_in_days} days") + if age_in_days > max_age_days: + client.deregister_image(ImageId=ami_id) + print(f"Deleted AMI {ami_id} in region {region} with age {age_in_days} days") diff --git a/boto3/ec2_information/retrieve_information_about_ec2.py b/boto3/ec2_information/retrieve_information_about_ec2.py new file mode 100644 index 0000000..b4d1577 --- /dev/null +++ b/boto3/ec2_information/retrieve_information_about_ec2.py @@ -0,0 +1,11 @@ +import boto3 +import csv + +ec2 = boto3.resource('ec2') +instances = ec2.instances.all() + +with open('ec2-instances.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(["Instance ID", "State", "AMI ID", "Platform", "Instance Type", "Public IPv4 Address"]) + for instance in instances: + writer.writerow([instance.id, instance.state['Name'], instance.image.id, instance.platform, instance.instance_type, instance.public_ip_address]) diff --git a/boto3/ec2_information/updating_tags.py b/boto3/ec2_information/updating_tags.py new file mode 100644 index 0000000..b041b07 --- /dev/null +++ b/boto3/ec2_information/updating_tags.py @@ -0,0 +1,25 @@ +import boto3 +import csv + +# Export list of EC2 instances to CSV file +ec2 = boto3.resource('ec2') +instances = ec2.instances.all() + +with open('ec2-instances.csv', 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(["Instance ID", "Name"]) + for instance in instances: + writer.writerow([instance.id, instance.tags[0]['Value'] if instance.tags else ""]) + +# Manually update CSV file with new tag keys and values + + +# Import updated CSV file and update tags for each EC2 instance +with open('ec2-instances-tags.csv', 'r') as file: + reader = csv.reader(file) + next(reader) # skip header row + for row in reader: + instance_id = row[0] + tag_key = row[2] + tag_value = row[3] + ec2.create_tags(Resources=[instance_id], Tags=[{'Key': tag_key, 'Value': tag_value}]) diff --git a/boto3/rotate_iam_keys/rotate_iam_key_v3.py b/boto3/rotate_iam_keys/rotate_iam_key_v3.py new file mode 100644 index 0000000..3244e16 --- /dev/null +++ b/boto3/rotate_iam_keys/rotate_iam_key_v3.py @@ -0,0 +1,41 @@ +import boto3 +from datetime import datetime, timezone + +client = boto3.client("iam") +s3 = boto3.client("s3") +paginator = client.get_paginator('list_users') + +max_key_age = 5 +s3_bucket = "my-bucket-name" +s3_key_prefix = "new-keys/" + +def rotate_key(key_creation_date): + current_date = datetime.now(timezone.utc) + age = (current_date - key_creation_date).days + return age + +for response in paginator.paginate(): + for user in response['Users']: + username = user['UserName'] + + # Create a new access key for the user + response = client.create_access_key(UserName=username) + access_key_id = response['AccessKey']['AccessKeyId'] + secret_access_key = response['AccessKey']['SecretAccessKey'] + + # Upload the new access key to an S3 bucket(Please do it at your own risk) + s3_key = f"{s3_key_prefix}{access_key_id}.txt" + s3.put_object( + Bucket=s3_bucket, + Key=s3_key, + Body=f"Access Key ID: {access_key_id}\nSecret Access Key: {secret_access_key}" + ) + + listkey = client.list_access_keys(UserName=username) + for accesskey in listkey['AccessKeyMetadata']: + accesskey_id = accesskey['AccessKeyId'] + key_creation_date = accesskey['CreateDate'] + age = rotate_key(key_creation_date) + if age > max_key_age: + print(f"Deactivating key for the following user: {username}") + client.update_access_key(UserName=username, AccessKeyId=accesskey_id, Status='Inactive') diff --git a/boto3/rotate_iam_keys/rotate_iam_keys_v4.py b/boto3/rotate_iam_keys/rotate_iam_keys_v4.py new file mode 100644 index 0000000..808d3b7 --- /dev/null +++ b/boto3/rotate_iam_keys/rotate_iam_keys_v4.py @@ -0,0 +1,30 @@ +import boto3 +from datetime import datetime, timezone + +client = boto3.client("iam") +paginator = client.get_paginator('list_users') + +max_key_age = 5 +excluded_users = ["user1", "user2"] # List of usernames to exclude + +def rotate_key(key_creation_date): + current_date = datetime.now(timezone.utc) + age = (current_date - key_creation_date).days + return age + +for response in paginator.paginate(): + for user in response['Users']: + username = user['UserName'] + + if username in excluded_users: + # Skip processing excluded users + continue + + listkey = client.list_access_keys(UserName=username) + for accesskey in listkey['AccessKeyMetadata']: + accesskey_id = accesskey['AccessKeyId'] + key_creation_date = accesskey['CreateDate'] + age = rotate_key(key_creation_date) + if age > max_key_age: + print(f"Deactivating key for the following user: {username}") + client.update_access_key(UserName=username, AccessKeyId=accesskey_id, Status='Inactive') diff --git a/ebs_volume_conversion/README.md b/ebs_volume_conversion/README.md new file mode 100644 index 0000000..d487bde --- /dev/null +++ b/ebs_volume_conversion/README.md @@ -0,0 +1,35 @@ +# Python script to convert EBS Volume + +Python script to convert EBS volume from gp2 to gp3 or from io1 to io2 and read input from csv file with a format volume_id,volume_type + +## Usage + +```python3 ebs_volume_conversion.py -h +usage: ebs_volume_conversion.py [-h] [-f FILE] volume_id volume_type + +positional arguments: + volume_id The EBS volume id of the Volume to be convert + volume_type The new EBS volume type to be converted gp3 or io2 + +optional arguments: + -h, --help show this help message and exit + -f FILE, --file FILE A csv file containing ebs volume id and volume type to convert +``` + +# Example + +1. To convert an ebs volume from gp2 to gp3 + +``` +python3 ebs_volume_conversion.py -volume_id -volume_type gp3 +``` + +2. To convert an ebs volume from io1 to io2 +``` +python3 ebs_volume_conversion.py -volume_id -volume_type io2 +``` + +3. You can pass a csv file containing ebs volume id and volume type to convert +``` +python3 ebs_volume_conversion.py -f ebs_volume.csv +``` diff --git a/ebs_volume_conversion/ebs_volume_conversion.py b/ebs_volume_conversion/ebs_volume_conversion.py new file mode 100644 index 0000000..c9ca367 --- /dev/null +++ b/ebs_volume_conversion/ebs_volume_conversion.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# Python script to convert EBS volume from gp2 to gp3 or from io1 to io2 and read input from csv file with a format volume_id,volume_type + +import boto3 +import argparse +import csv +import sys + +def convert_ebs_vol(volume_id,volume_type): + # Create an EC2 client + ec2 = boto3.client('ec2') + # Ask user if he want to take the snapshot before volume conversion + snapshot = input(f'Do you want to take the snapshot of the following volume {volume_id} before conversion? (y/n)') + # Take the snapshot if user confirms + if snapshot.lower() == 'y': + try: + snapshot = ec2.create_snapshot(VolumeId=volume_id,Description="Snapshot before converting the EBS volume") + # Wait for the snapshot to be completed + waiter = ec2.get_waiter('snapshot_completed') + waiter.wait(SnapshotIds=[snapshot['SnapshotId']]) + print(f'Sucessfully created the snapshot {snapshot["SnapshotId"]} of volume {volume_id}') + except Exception as e: + print(f'An error occurred while creating the snapshot for volume {volume_id}: {e} ') + exit(1) + try: + # Modify the EBS Volume to the new volume type + ec2.modify_volume(VolumeId=volume_id, VolumeType=volume_type) + print(f'Started converting EBS volume {volume_id} to {volume_type}') + # Wait until the EBS volume is available + ec2.get_waiter('volume_available').wait(VolumeIds=[volume_id]) + print(f'Sucessfully converted EBS volume {volume_id} to {volume_type}') + except Exception as e: + print(f'An error occurred while converting the EBS volume {volume_id}: {e} ') + exit(1) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-volume_id", help="The EBS volume id of the Volume to be convert") + parser.add_argument("-volume_type", help="The new EBS volume type to be converted gp3 or io2") + parser.add_argument("-f","--file", help="A csv file containing ebs volume id and volume type to convert") + args = parser.parse_args() + if not any(vars(args).values()): + parser.print_help() + sys.exit() + + if args.file: + with open(args.file, 'r') as f: + reader = csv.reader(f) + for row in reader: + volume_id = row[0] + volume_type = row[1] + convert_ebs_vol(volume_id, volume_type) + else: + convert_ebs_vol(args.volume_id, args.volume_type) diff --git a/ec2_instance_conversion/README.md b/ec2_instance_conversion/README.md new file mode 100644 index 0000000..882e89c --- /dev/null +++ b/ec2_instance_conversion/README.md @@ -0,0 +1,35 @@ +# Python script to convert EC2 instance + +Python script to convert EC2 instance from one type to another and read input from csv file with a format instance_id,instance_type + + +## Usage + +```python3 ec2_instance_conversion.py +usage: ec2_instance_conversion.py [-h] [-instance_id INSTANCE_ID] + [-new_instance_type NEW_INSTANCE_TYPE] [-f FILE] + +Convert an EC2 instance from one type to another + +optional arguments: + -h, --help show this help message and exit + -instance_id INSTANCE_ID + The ID of the EC2 instance to convert + -new_instance_type NEW_INSTANCE_TYPE + The new instance type for the EC2 instance + -f FILE, --file FILE A CSV file containing a list of instances and new instance types +``` + +# Example + +1. To convert an EC2 instance to t2.medium + +``` +python3 ec2_instance_conversion.py -instance_id -new_instance_type t2.medium +``` + + +3. You can pass a csv file containing instance id and instance type to convert +``` +python3 ec2_instance_conversion.py -f ec2_conversion.csv +``` diff --git a/ec2_instance_conversion/ec2_instance_conversion.py b/ec2_instance_conversion/ec2_instance_conversion.py new file mode 100644 index 0000000..c9134f8 --- /dev/null +++ b/ec2_instance_conversion/ec2_instance_conversion.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Python script to convert EC2 instance from one type to another and read input from csv file with a format instance_id,instance_type + +import argparse +import boto3 +import sys +import csv + +def convert_instance(instance_id, new_instance_type, csv_file=None): + # Create an EC2 client + ec2 = boto3.client('ec2') + + # Check if the instance is in a stopped state + response = ec2.describe_instances(InstanceIds=[instance_id]) + instance_state = response['Reservations'][0]['Instances'][0]['State']['Name'] + if instance_state == 'stopped': + snapshot_response = input("The instance is in a stopped state. Please don't forget to take the ebs snapshot of the instance volume before the conversion y/n") + try: + # Modify the instance + ec2.modify_instance_attribute(InstanceId=instance_id, Attribute='instanceType', Value=new_instance_type) + print("Successfully converted instance", instance_id, "to", new_instance_type) + except Exception as e: + print("Error converting instance:", e) + +if __name__ == '__main__': + # Set up the argument parser + parser = argparse.ArgumentParser(description='Convert an EC2 instance from one type to another') + parser.add_argument('-instance_id', help='The ID of the EC2 instance to convert') + parser.add_argument('-new_instance_type', help='The new instance type for the EC2 instance') + parser.add_argument("-f","--file", help='A CSV file containing a list of instances and new instance types') + args = parser.parse_args() + if not any(vars(args).values()): + parser.print_help() + sys.exit() + if args.file: + with open(args.file, 'r') as f: + reader = csv.reader(f) + for row in reader: + instance_id = row[0] + new_instance_type = row[1] + convert_instance(instance_id, new_instance_type) + else: + convert_instance(args.instance_id, args.new_instance_type) diff --git a/environment_setup/environment_setup.sh b/environment_setup/environment_setup.sh new file mode 100644 index 0000000..a4651b0 --- /dev/null +++ b/environment_setup/environment_setup.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# This shell script that checks if the required python modules (pip3, boto3, and awscli) +# the terraform package are installed, and installs them if they are missing. +# This script should work on both CentOS and macOS +# Check the terraform release page to use the latest terraform package https://releases.hashicorp.com/terraform/ +# Author: Prashant Lakhera(laprashant@gmail.com) + +# Check if pip3 is installed +if ! type "pip3" &> /dev/null +then + echo "pip3 is not installed. Installing now ..." + # Install pip3 + # Checking for Mac + if [ "$(uname)" == "Darwin" ] + then + brew install python3 + # Checking for Centos + elif [ -f /etc/centos-release ] + then + sudo yum -y update + sudo yum -y install python3-pip -y + fi +fi + +# Check if boto3 is installed +if python3 -c "import boto3" &> /dev/null; then + echo "Boto3 is already installed" +else + echo "Boto3 is not installed. Installing now...." + pip3 install boto3 --user +fi + +# Check if awscli is installed +if ! type "aws" > /dev/null +then + echo "awscli is not installed. Installing now..." + pip3 install awscli --user +else + echo "awscli is already installed" +fi + +# Check if terraform is installed +if ! type "terraform" > /dev/null +then + echo "terraform is not installed installing now..." + # Install terraform + #Checking for Mac + if [ "$(uname)" == "Darwin" ] + then + brew install terraform + # Checking for Centos + elif [ -f /etc/centos-release ] + then + sudo yum install -y unzip + sudo wget https://releases.hashicorp.com/terraform/1.3.6/terraform_1.3.6_linux_amd64.zip + sudo unzip terraform_1.3.6_linux_amd64.zip + sudo mv terraform /usr/local/bin/ + sudo chmod +x /usr/local/bin/terraform + sudo rm terraform_1.3.6_linux_amd64.zip + fi +else + echo "Terraform is already installed" + +fi \ No newline at end of file diff --git a/github_automation/downloading_all_repos.py b/github_automation/downloading_all_repos.py new file mode 100644 index 0000000..2e55195 --- /dev/null +++ b/github_automation/downloading_all_repos.py @@ -0,0 +1,25 @@ +import requests +import os + +# Replace with your GitHub username and personal access token +username = "your-username" +access_token = "your-access-token" + +# Set the base URL for the GitHub API +base_url = "https://api.github.com" + +# Create a session with the access token to authenticate requests +session = requests.Session() +session.auth = (username, access_token) + +# Get the list of repositories for the authenticated user +repositories_url = f"{base_url}/user/repos" +response = session.get(repositories_url) +repositories = response.json() + +# Loop over the repositories and clone them to a local directory +for repository in repositories: + name = repository["name"] + clone_url = repository["clone_url"] + print(f"Cloning {name} from {clone_url}") + os.system(f"git clone {clone_url}") diff --git a/kubernetes_debugging/container_creating_error.sh b/kubernetes_debugging/container_creating_error.sh new file mode 100755 index 0000000..468338a --- /dev/null +++ b/kubernetes_debugging/container_creating_error.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Script to check ContainerCreating error in Kubernetes pods +# Author: Prashant Lakhera(laprashant@gmail.com) + +# Set the namespace which we use to query the pods +namespace="default" + +# Query for the pods using the namespace +pods=$(kubectl get pods -n "$namespace" -o json | jq -r '.items[].metadata.name') + +# Iterate through the list of pods +for pod in $pods; do + # Get the status of the current pod + status=$(kubectl get pod "$pod" -n "$namespace" -o json | jq -r '.status.containerStatuses[].state.waiting.reason') + # Check if the status is "ContainerCreating" + if [ "$status" == "ContainerCreating" ]; then + # Print an error message + echo "ERROR: Pod $pod is in a ContainerCreating state!" + exit 1 + # In some cases this is helpful,try to delete the pod to allow it to be rescheduled + # kubectl delete pod "$pod" -n "$namespace" + fi +done + + diff --git a/kubernetes_debugging/crash_loopback_error.sh b/kubernetes_debugging/crash_loopback_error.sh new file mode 100755 index 0000000..6841637 --- /dev/null +++ b/kubernetes_debugging/crash_loopback_error.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Script to check Crashloopbackoff error in Kubernetes pods +# Author: Prashant Lakhera(laprashant@gmail.com) + +# Set the namespace which we use to query the pods +namespace="default" + +# Query for the pods using the namespace +pods=$(kubectl get pods -n "$namespace" -o json | jq -r '.items[].metadata.name') + +# Iterate through the list of pods +for pod in $pods; do + # Get the status of the current pod + status=$(kubectl describe pod "$pod" -n "$namespace"| grep "CrashLoopBackOff" |awk '{print $2}') + + # If the status is "CrashLoopBackOff", handle the error + for state in status + do + if [ "$status" == "CrashLoopBackOff" ]; then + # Print an error message + echo "ERROR: Pod $pod is in a CrashLoopBackOff state!" + exit 1 + + # Try to delete the pod to allow it to be rescheduled + #kubectl delete pod "$pod" -n "$namespace" + + # Print a message indicating that the pod was deleted + #echo "INFO: Deleted pod $pod to allow it to be rescheduled." + fi + done +done diff --git a/kubernetes_debugging/create_container_config.sh b/kubernetes_debugging/create_container_config.sh new file mode 100755 index 0000000..fdc2a3d --- /dev/null +++ b/kubernetes_debugging/create_container_config.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Script to check ContainerCreating error in Kubernetes pods +# Author: Prashant Lakhera(laprashant@gmail.com) + +# Set the namespace which we use to query the pods +namespace="default" + +# Query for the pods using the namespace +pods=$(kubectl get pods -n "$namespace" -o json | jq -r '.items[].metadata.name') + +# Iterate through the list of pods +for pod in $pods; do + # Get the status of the current pod + status=$(kubectl describe pod "$pod" -n "$namespace" | grep "CreateContainerConfigError" | awk '{print $2}') + # If the status of the pod contains the "CreateContainerConfig" error + if [[ "$status" = "CreateContainerConfigError" ]]; then + # Print an error message + echo "ERROR: Pod $pod has a CreateContainerConfig error!" + exit 1 + + # In some cases,Try to delete the pod to allow it to be rescheduled + # kubectl delete pod "$pod" -n "$namespace" + fi +done \ No newline at end of file diff --git a/kubernetes_debugging/kubernetes_debugging_select.sh b/kubernetes_debugging/kubernetes_debugging_select.sh new file mode 100755 index 0000000..ec77de5 --- /dev/null +++ b/kubernetes_debugging/kubernetes_debugging_select.sh @@ -0,0 +1,31 @@ +# Create a list of options for Kubernetes Debugging +options=("CrashLoopBackOff" "ImagePullBackOff" "ContainerCreating" "CreateContainerConfigError" "Quit") + +# Display the menu and prompt the user to select an option +echo "Please select from the following Kubernetes Debugging option" +PS3="Please select an option: " +select opt in "${options[@]}"; do + # Process the user's selection + case $opt in + "CrashLoopBackOff") + # Execute crash_loopback_error.sh + source crash_loopback_error.sh + ;; + "ImagePullBackOff") + # Execute pod_image_pull_error.sh + source pod_image_pull_error.sh + ;; + "ContainerCreating") + # Execute container_creating_error.sh + source container_creating_error.sh + ;; + "CreateContainerConfigError") + # Execute create_container_config.sh + source create_container_config.sh + ;; + "Quit") + break + ;; + *) echo "Invalid option. Please try again.";; + esac +done diff --git a/kubernetes_debugging/pod_image_pull_error.sh b/kubernetes_debugging/pod_image_pull_error.sh new file mode 100755 index 0000000..630eabd --- /dev/null +++ b/kubernetes_debugging/pod_image_pull_error.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Script to check image pull error in Kubernetes pods +# Author: Prashant Lakhera(laprashant@gmail.com) + +# Set the namespace which we use to query the pods +NAMESPACE=default + +# Query for the pods using the namespace and label selector +PODS=$(kubectl get pods -n $NAMESPACE -o json) + +# Verify if the query return any pods +if [ -z "$PODS" ]; then + echo "No pods found" + exit 0 +fi + +# Parse the JSON output to get the list of pod names +POD_NAMES=$(echo $PODS | jq -r '.items[].metadata.name') + +# Iterate over the list of pod names +for POD_NAME in $POD_NAMES; do + # Check for image pull errors in the pod + IMAGE_PULL_ERRORS=$(kubectl describe pod $POD_NAME -n $NAMESPACE | grep -c "ImagePullBackOff") + + # If there is a image pull errors, print an error message and exit with an error code + if [ "$IMAGE_PULL_ERRORS" -gt 0 ]; then + echo "There were image pull errors in pod $POD_NAME" + fi +done + +# If there is no image pull error and all the pods are running, exit with a zero status code +exit 0 \ No newline at end of file diff --git a/performance_tuning/performance-monitoring-simplified.sh b/performance_tuning/performance-monitoring-simplified.sh new file mode 100644 index 0000000..7984f04 --- /dev/null +++ b/performance_tuning/performance-monitoring-simplified.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# This script is the quick check to detect whether the performance issue is due to CPU, Memory, Input/Output (I/O), and network error +# This script should work on both CentOS and macOS +# Author: Prashant Lakhera(laprashant@gmail.com) + + +# Check if the load average is greater than 70% of the CPU cores +load_avg=$(w | head -n 1 | awk '{print $9}' |cut -f1 -d",") +num_cores=$(nproc) +max_load=$(echo "0.7 * $num_cores" | bc) + +if [[ $(echo "$load_avg > $max_load" | bc) -eq 1 ]]; then + #Print a message if the load average is too high + echo -e "\033[1;31m CPU load average is currently $load_avg, which is higher than the maximum of $max_load \033[0m" >&2 +else + # Print a message if the load average is within the acceptable range + echo -e "\033[1;32m CPU load average is currently $load_avg, which is within the acceptable range.\033[0m" +fi + +# Set the memory average threshold +THRESHOLD=90 + +# Get the total memory and used memory in bytes +total_memory=$(grep 'MemTotal' /proc/meminfo | awk '{print $2}') +available_memory=$(grep 'MemAvailable' /proc/meminfo | awk '{print $2}') + +# Calculate the actual memory utilization as a percentage +memory_utilization=$(echo "scale=2; ($total_memory - $available_memory)/$total_memory * 100" | bc) + +# Compare the memory utilization with the threshold +if (( $(echo "$memory_utilization > $THRESHOLD" | bc -l) )) +then + echo -e "\033[1;32m Memory utilization is above the threshold!!! Memory utilization is: $utilization% \033[0m" +else + echo -e "\033[1;32m Memory utilizationis currently $memory_utilization, which is within the acceptable range.\033[0m" +fi + +# Check the I/O wait state + +iowait_state=$(top -b -n 1 | head -n +3|awk '{print $10}'|tail -1 |bc) +if [[ $(echo "$iowait_state > 1" | bc) -eq 1 ]]; then + #Print a message IOWAIT is too high + echo -e "\033[1;31m IOWAIT is currently $iowait_state, which is higher than the acceptable range \033[0m" >&2 +else + # Print a message IOWAIT is within the acceptable range + echo -e "\033[1;32m IOWAIT is currently $iowait_state, which is within the acceptable range.\033[0m" +fi + +# Check if ifconfig command is present +if command -v ifconfig >/dev/null 2>&1; then + echo "ifconfig command is present" +else + echo "ifconfig command is not present. Installing..." + # Install ifconfig command + if [ -f /etc/centos-release ]; then + # CentOS + sudo yum install -y net-tools + elif [ -f /etc/lsb-release ]; then + # Ubuntu + sudo apt-get update + sudo apt-get install -y net-tools + else + # Unsupported OS + echo "Unsupported operating system" + exit 1 + fi +fi + +#Get the network interface name or ask input from the user +#interface=$1 +interface=$(ifconfig |head -1|awk '{print $1}' |cut -f1 -d:) + +# Get the RX error count +rx_error_count=$(ifconfig $interface | grep "RX errors" |awk '{print $3}') + +# Get the TX error count +tx_error_count=$(ifconfig $interface | grep "TX errors" |awk '{print $3}') + +# Check if either error count is greater than zero +# Remember these counter only get reset after reboot, so you may get some false alarm. Check this thread for more reference https://unix.stackexchange.com/questions/164057/how-can-i-manually-reset-rx-tx-counters-in-ifconfig-output-without-impacting-d +if [[ $rx_error_count -gt 0 || $tx_error_count -gt 0 ]]; then + #Print a message Network error count is too high + echo -e "\033[1;31m Network Error is currently for Revieve Error: $rx_error_count and Transmit Error: $tx_error_count, which is higher than the acceptable range \033[0m" >&2 +else + # Print a message Network error count is within the acceptable range + echo -e "\033[1;32m Network Error is currently for Revieve Error: $rx_error_count and Transmit Error: $tx_error_count, which is within the acceptable range.\033[0m" +fi \ No newline at end of file diff --git a/performance_tuning/performance-monitoring-v2.sh b/performance_tuning/performance-monitoring-v2.sh new file mode 100644 index 0000000..c76b204 --- /dev/null +++ b/performance_tuning/performance-monitoring-v2.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# This script list the top 5 CPU, memory, input/output (I/O), and network consuming processes +# This script should work on both CentOS and macOS +# Author: Prashant Lakhera(laprashant@gmail.com) + + +# Function to list top 5 CPU consuming process +list_top_cpu_consuming_processes() { + # Get the top 5 processes by CPU usage + top_five_cpu_processes=$(ps -eo pcpu,pid,user,args --no-headers | sort -k 1 -r | head -n 5) + + # Print the results + echo "###############################################################################" + echo "Top 5 CPU consuming processes:" + echo "$top_five_cpu_processes" +} + +# Function to list the top 5 processes by memory usage +list_top_memory_consuming_processes() { + # Get the top 5 processes by memory usage + top_five_memory_processes=$(ps -eo pmem,pid,user,args --no-headers | sort -k 1 -r | head -n 5) + + # Print the results + echo "###############################################################################" + echo "Top 5 memory consuming processes:" + echo "$top_five_memory_processes" +} + +# Check if iotop command is present +if command -v iotop >/dev/null 2>&1; then + echo "iotop command is present" +else + echo "iotop command is not present. Installing..." + # Install top command + if [ -f /etc/centos-release ]; then + # CentOS + sudo yum install -y epel-release + sudo yum install -y iotop + elif [ -f /etc/lsb-release ]; then + # Ubuntu + sudo apt-get update + sudo apt-get install -y iotop + else + # Unsupported OS + echo "Unsupported operating system" + exit 1 + fi +fi + +# Function to list the top 5 processes by I/O usage +list_top_io_consuming_processes() { + # Get the top 5 processes by I/O usage + top_five_io_processes=$(sudo iotop -o -b -n 5) + + # Print the results + echo "###############################################################################" + echo "Top 5 I/O consuming processes:" + echo "$top_five_io_processes" +} + +# Check if iftop command is present +if command -v iftop >/dev/null 2>&1; then + echo "iftop command is present" +else + echo "iftop command is not present. Installing..." + # Install top command + if [ -f /etc/centos-release ]; then + # CentOS + sudo yum install -y iftop + elif [ -f /etc/lsb-release ]; then + # Ubuntu + sudo apt-get update + sudo apt-get install -y iftop + else + # Unsupported OS + echo "Unsupported operating system" + exit 1 + fi +fi + + +# Function to list the top 5 processes by network usage +list_top_network_consuming_processes() { + # Get the top 5 processes by network usage + top_five_network_processes=$(sudo iftop -P -n -t -s 5) + + # Print the results + echo "###############################################################################" + echo "Top 5 network consuming processes:" + echo "$top_five_network_processes" +} + + +# Main function +main() { + list_top_cpu_consuming_processes + list_top_memory_consuming_processes + list_top_io_consuming_processes + list_top_network_consuming_processes +} + +# Run the main function +main diff --git a/performance_tuning/performance-monitoring.py b/performance_tuning/performance-monitoring.py new file mode 100644 index 0000000..7439e5d --- /dev/null +++ b/performance_tuning/performance-monitoring.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +import os + +# Function to list top 5 CPU consuming processes +def list_top_cpu_consuming_processes(): + # Get the top 5 processes by CPU usage + top_five_cpu_processes = os.popen("ps -eo pcpu,pid,user,args | sort -k 1 -r | head -n 6").read() + + # Print the results + print("###############################################################################") + print("Top 5 CPU consuming processes:") + print(top_five_cpu_processes) + +# Function to list the top 5 processes by memory usage +def list_top_memory_consuming_processes(): + # Get the top 5 processes by memory usage + top_five_memory_processes = os.popen("ps -eo pmem,pid,user,args | sort -k 1 -r | head -n 6").read() + + # Print the results + print("###############################################################################") + print("Top 5 memory consuming processes:") + print(top_five_memory_processes) + +# Check if iotop command is present +if os.system("command -v iotop >/dev/null 2>&1") == 0: + print("iotop command is present") +else: + print("iotop command is not present. Installing...") + # Install iotop command + if os.path.isfile("/etc/centos-release"): + # CentOS + os.system("sudo yum install -y epel-release") + os.system("sudo yum install -y iotop") + elif os.path.isfile("/etc/lsb-release"): + # Ubuntu + os.system("sudo apt-get update") + os.system("sudo apt-get install -y iotop") + else: + # Unsupported OS + print("Unsupported operating system") + exit(1) + +# Function to list the top 5 processes by I/O usage +def list_top_io_consuming_processes(): + # Get the top 5 processes by I/O usage + top_five_io_processes = os.popen("sudo iotop -o -b -n 6").read() + + # Print the results + print("###############################################################################") + print("Top 5 I/O consuming processes:") + print(top_five_io_processes) + +# Check if iftop command is present +if os.system("command -v iftop >/dev/null 2>&1") == 0: + print("iftop command is present") +else: + print("iftop command is not present. Installing...") + # Install iftop command + if os.path.isfile("/etc/centos-release"): + # CentOS + os.system("sudo yum install -y iftop") + elif os.path.isfile("/etc/lsb-release"): + # Ubuntu + os.system("sudo apt-get update") + os.system("sudo apt-get install -y iftop") + else: + # Unsupported OS + print("Unsupported operating system") + exit(1) + +# Function to list the top 5 processes by network usage +def list_top_network_consuming_processes(): + # Get the top 5 processes by network usage + top_five_network_processes = os.popen("sudo iftop -P -n -t -s 6").read() + + # Print the results + print("###############################################################################") + print("Top 5 network consuming processes:") + print(top_five_network_processes) + +# Main function +def main(): + list_top_cpu_consuming_processes() + list_top_memory_consuming_processes() + list_top_io_consuming_processes() + list_top_network_consuming_processes() + +# Run the main function +if __name__ == "__main__": + main() diff --git a/performance_tuning/performance-monitoring.sh b/performance_tuning/performance-monitoring.sh new file mode 100644 index 0000000..7d74976 --- /dev/null +++ b/performance_tuning/performance-monitoring.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# This script list the top 5 CPU, memory, input/output (I/O), and network consuming processes +# This script should work on both CentOS and macOS +# Author: Prashant Lakhera(laprashant@gmail.com) + + +# Function to list top 5 CPU consuming process +list_top_cpu_consuming_processes() { + # Get the top 5 processes by CPU usage + top_five_cpu_processes=$(ps -eo pcpu,pid,user,args | sort -k 1 -r | head -n 6) + + # Print the results + echo "###############################################################################" + echo "Top 5 CPU consuming processes:" + echo "$top_five_cpu_processes" +} + +# Function to list the top 5 processes by memory usage +list_top_memory_consuming_processes() { + # Get the top 5 processes by memory usage + top_five_memory_processes=$(ps -eo pmem,pid,user,args | sort -k 1 -r | head -n 6) + + # Print the results + echo "###############################################################################" + echo "Top 5 memory consuming processes:" + echo "$top_five_memory_processes" +} + +# Check if iotop command is present +if command -v iotop >/dev/null 2>&1; then + echo "iotop command is present" +else + echo "iotop command is not present. Installing..." + # Install top command + if [ -f /etc/centos-release ]; then + # CentOS + sudo yum install -y epel-release + sudo yum install -y iotop + elif [ -f /etc/lsb-release ]; then + # Ubuntu + sudo apt-get update + sudo apt-get install -y iotop + else + # Unsupported OS + echo "Unsupported operating system" + exit 1 + fi +fi + +# Function to list the top 5 processes by I/O usage +list_top_io_consuming_processes() { + # Get the top 5 processes by I/O usage + top_five_io_processes=$(sudo iotop -o -b -n 6) + + # Print the results + echo "###############################################################################" + echo "Top 5 I/O consuming processes:" + echo "$top_five_io_processes" +} + +# Check if iftop command is present +if command -v iftop >/dev/null 2>&1; then + echo "iftop command is present" +else + echo "iftop command is not present. Installing..." + # Install top command + if [ -f /etc/centos-release ]; then + # CentOS + sudo yum install -y iftop + elif [ -f /etc/lsb-release ]; then + # Ubuntu + sudo apt-get update + sudo apt-get install -y iftop + else + # Unsupported OS + echo "Unsupported operating system" + exit 1 + fi +fi + + +# Function to list the top 5 processes by network usage +list_top_network_consuming_processes() { + # Get the top 5 processes by network usage + top_five_network_processes=$(sudo iftop -P -n -t -s 6) + + # Print the results + echo "###############################################################################" + echo "Top 5 network consuming processes:" + echo "$top_five_network_processes" +} + + +# Main function +main() { + list_top_cpu_consuming_processes + list_top_memory_consuming_processes + list_top_io_consuming_processes + list_top_network_consuming_processes +} + +# Run the main function +main