diff --git a/.gitignore b/.gitignore index 808f4df3..8711b0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -app/api/scan_tests/testing_stuff.py +app/api/utils/testing_stuff.py app/data* -app/api/scan_tests/__pycache__/tester.cpython-38.pyc +app/api/utils/__pycache__/tester.cpython-38.pyc .DS_Store *__pycache__* db.sqlite3 @@ -14,3 +14,6 @@ env/.env.staging env/.env.dev env/.env.prod env/.env.prod.db +app/static* + +app/api/migrations/0001_initial.py diff --git a/Dockerfile b/Dockerfile index d4488583..3e69e8a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,23 @@ -FROM python:3.8-alpine +FROM python:3.9-alpine ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user RUN addgroup -S app && adduser -S app -G app +# installing postgres deps RUN apk add --update --no-cache postgresql-client jpeg-dev +# installing python3 +RUN apk add --no-cache --update \ + python3 python3-dev + +# installing env deps RUN apk add --update --no-cache --virtual .tmp-build-deps \ gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl + build-base libffi libffi-dev fontconfig libjpeg-turbo-dev \ + ttf-freefont ca-certificates freetype freetype-dev harfbuzz nss \ + nasm git make g++ automake autoconf libtool gfortran openssl # installing chromium and chromium-chromedriver RUN apk add --update --no-cache chromium chromium-chromedriver @@ -21,16 +25,50 @@ RUN apk add --update --no-cache chromium chromium-chromedriver # installing node and npm RUN apk add --update nodejs npm +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + # installing lighthouse RUN npm install -g lighthouse +# installing yellowlabs tools +RUN npm install -g yellowlabtools + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium-browser --no-sandbox --version + +# inatalling numpy +RUN apk add --update --no-cache py3-numpy +# installing scipy +RUN apk add --update --no-cache py3-scipy + +# setting path for numpy and scipy +ENV PYTHONPATH /usr/lib/python3.9/site-packages + +# super hacky BS to fix Alpine instalation issues with the data science packages +RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; + +# installing sewar +RUN python3 -m pip install --no-deps sewar==0.4.4 + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt +RUN apk del .tmp-build-deps + +# setting working dir RUN mkdir /app COPY ./app /app WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app:app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium-browser \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod index e8a614be..3e69e8a4 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,19 +1,23 @@ -FROM python:3.8-alpine +FROM python:3.9-alpine ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user RUN addgroup -S app && adduser -S app -G app +# installing postgres deps RUN apk add --update --no-cache postgresql-client jpeg-dev +# installing python3 +RUN apk add --no-cache --update \ + python3 python3-dev + +# installing env deps RUN apk add --update --no-cache --virtual .tmp-build-deps \ gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl + build-base libffi libffi-dev fontconfig libjpeg-turbo-dev \ + ttf-freefont ca-certificates freetype freetype-dev harfbuzz nss \ + nasm git make g++ automake autoconf libtool gfortran openssl # installing chromium and chromium-chromedriver RUN apk add --update --no-cache chromium chromium-chromedriver @@ -21,16 +25,50 @@ RUN apk add --update --no-cache chromium chromium-chromedriver # installing node and npm RUN apk add --update nodejs npm +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + # installing lighthouse RUN npm install -g lighthouse +# installing yellowlabs tools +RUN npm install -g yellowlabtools + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium-browser --no-sandbox --version + +# inatalling numpy +RUN apk add --update --no-cache py3-numpy + +# installing scipy +RUN apk add --update --no-cache py3-scipy + +# setting path for numpy and scipy +ENV PYTHONPATH /usr/lib/python3.9/site-packages + +# super hacky BS to fix Alpine instalation issues with the data science packages +RUN find /usr/lib/python3.9/site-packages -iname "*.so" -exec sh -c 'x="{}"; mv "$x" "${x/cpython-39-x86_64-linux-musl./}"' \; + +# installing sewar +RUN python3 -m pip install --no-deps sewar==0.4.4 + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt +RUN apk del .tmp-build-deps + +# setting working dir RUN mkdir /app COPY ./app /app -RUN rm -rf /app/static; mkdir /app/static WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium-browser \ No newline at end of file diff --git a/README.md b/README.md index b65da111..3b18629b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Copyright © Scanerr 2021 - [Environment](#environment) - [Local](#local) - [Remote](#remote) + - [Scripts](#scripts)   @@ -29,13 +30,16 @@ Copyright © Scanerr 2021 ## Environment -Prior to running locally, configure all env's located in the /env directory. **For remote deployment, create all .env's in the deployed environment. Never store actual .env's in a repo.** Things to change: -- email addresses -- passwords -- usernames -- allowed hosts -- external API services and keys -- domain names (server and client) +Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: +- high level django configs +- admin credentials +- email credentials +- database configs +- stripe keys +- OAuth keys +- twilio credentials +- slack credentials +- s3 remote storage credentials   @@ -102,4 +106,18 @@ $ docker-compose -f docker-compose.prod.yml down *Spin-down the application and removes the volumes* ```shell $ docker-compose -f docker-compose.prod.yml down -v -``` \ No newline at end of file +``` + + +  + +--- + +  + +## Scripts + +*ssh into container* +``` shell +$ docker exec -it /bin/sh +``` diff --git a/app/api/admin.py b/app/api/admin.py index 4d664019..76f4df26 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,8 +1,6 @@ from django.contrib import admin -from .models import ( - Site, Test, Scan, Account, - Card, Log, Schedule, Automation -) +from .models import * + @admin.register(Site) class SiteAdmin(admin.ModelAdmin): @@ -29,6 +27,10 @@ class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) +@admin.register(Report) +class ReportAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'user') + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py index 02761f61..ec029a53 100644 --- a/app/api/management/commands/driver_test.py +++ b/app/api/management/commands/driver_test.py @@ -1,7 +1,5 @@ -from ...scan_tests.driver_init import driver_init +from ...utils.driver import driver_init from django.core.management.base import BaseCommand -from selenium import webdriver -from selenium.webdriver.chrome.options import Options import time, os, sys # testing selenium, chromedriver, and chromium installation and configs diff --git a/app/api/models.py b/app/api/models.py index 1e59c360..6ecd1262 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -11,51 +11,129 @@ def get_info_default(): info_default = { 'latest_scan': { - 'id': '', - 'time_created': '', + 'id': None, + 'time_created': None, }, 'latest_test': { - 'id': '', - 'time_created': '', - 'score': '' + 'id': None, + 'time_created': None, + 'score': None }, 'lighthouse': { - 'average': '', - 'seo': '', - 'performance': '', - 'accessibility': '', - 'best_practices': '', + 'average': None, + 'seo': None, + 'performance': None, + 'accessibility': None, + 'best_practices': None, + }, + 'yellowlab': { + 'globalScore': None, + 'pageWeight': None, + 'requests': None, + 'domComplexity': None, + 'javascriptComplexity': None, + 'badJavascript': None, + 'jQuery': None, + 'cssComplexity': None, + 'badCSS': None, + 'fonts': None, + 'serverConfig': None, }, 'status': { - 'ping': '', - 'health': '', + 'ping': None, + 'health': None, 'badge': 'neutral', + 'score': None, }, } return info_default -def get_scores_delta_default(): - scores_delta_default = { - "seo_delta": None, - "current_average": None, - "performance_delta": None, - "accessibility_delta": None, - "best_practices_delta": None +def get_lh_delta_default(): + lh_delta_default = { + "scores": { + "seo_delta": None, + "performance_delta": None, + "accessibility_delta": None, + "best-practices_delta": None, + "average_delta" : None, + "current_average": None, + }, } - return scores_delta_default - - - -def get_audits_default(): - audits_default = { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [] + return lh_delta_default + + + +def get_yl_delta_default(): + yl_delta_default = { + "scores": { + "globalScore_delta": None, + "pageWeight_delta": None, + "requests_delta": None, + "domComplexity_delta": None, + "javascriptComplexity_delta": None, + "badJavascript_delta": None, + "jQuery_delta": None, + "cssComplexity_delta": None, + "badCSS_delta": None, + "fonts_delta": None, + "serverConfig_delta": None, + }, + } + return yl_delta_default + + + +def get_lh_default(): + lh_default = { + "scores": { + "seo": None, + "performance": None, + "accessibility": None, + "best_practices": None, + "average": None, + }, + "audits": { + "seo": [], + "performance": [], + "accessibility": [], + "best-practices": [] + }, + } + return lh_default + + + +def get_yl_default(): + yl_default = { + "scores": { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + }, + "audits": { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + }, } - return audits_default + return yl_default @@ -73,6 +151,7 @@ def get_expressions_default(): return expressions_default + def get_actions_default(): actions_default = { 'list': [ @@ -89,6 +168,7 @@ def get_actions_default(): return actions_default + def get_slack_default(): slack_default = { "slack_name": None, @@ -102,6 +182,8 @@ def get_slack_default(): + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) @@ -121,8 +203,10 @@ class Scan(models.Model): time_created = models.DateTimeField(default=timezone.now, serialize=True) html = models.TextField(serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) - scores = models.JSONField(serialize=True, null=True, blank=True) - audits = models.JSONField(serialize=True, null=True, blank=True, default=get_audits_default) + images = models.JSONField(serialize=True, null=True, blank=True) + lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) + yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) + configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__scan' @@ -134,13 +218,15 @@ class Test(models.Model): site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # (1) html (2) error_logs TODO decide on this attr + type = models.JSONField(serialize=True, null=True, blank=True) pre_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='pre_scan') post_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) - scores_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) + lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) + yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) + images_delta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__test' @@ -198,6 +284,7 @@ class Schedule(models.Model): crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__{self.task_type}' @@ -216,6 +303,22 @@ class Automation(models.Model): def __str__(self): return f'{self.name}' + + + + + +class Report(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab, crux] + info = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.site.site_url}__report' diff --git a/app/api/scan_tests/alerts.py b/app/api/scan_tests/alerts.py deleted file mode 100644 index 2d6288f9..00000000 --- a/app/api/scan_tests/alerts.py +++ /dev/null @@ -1,329 +0,0 @@ -from django.core.mail import send_mail, send_mass_mail -from django.contrib.auth.models import User -from django.template.loader import render_to_string -from datetime import date -import os, operator, json, requests, uuid -from django.utils.html import strip_tags -from django.contrib.auth.models import User -from rest_framework.response import Response -from ..models import (Schedule, Automation, Test, Scan, Site, Account) -from twilio.rest import Client -from slack_sdk.web import WebClient -from slack_sdk.errors import SlackApiError - - - - - -def automation_email(email=None, automation_id=None, scan_or_test_id=None): - if email and automation_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ('\t'+''.join(exp_list)) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - subject = f'Alert for {site.site_url}' - title = f'Alert for {site.site_url}' - pre_header = f'Alert for {site.site_url}' - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - subject = subject - context = { - 'title' : title, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'exp_list': exp_list, - 'object_url' : object_url, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'View Site Dashboard', - 'content' : content, - 'signature' : '- Cheers!', - } - - html_message = render_to_string('api/automation_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - - - - -def automation_webhook( - request_type=None, - request_url=None, - request_data=None, - automation_id=None, - scan_or_test_id=None, - ): - if request_type and automation_id and request_url and request_data and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - json_data = json.loads(request_data) - get_list = ['?',] - - for key in json_data: - if 'test_score' == json_data[key]: - json_data[key] = item.score - elif 'current_average' == json_data[key]: - json_data[key] = item.scores_delta["current_average"] - elif 'seo_delta' == json_data[key]: - json_data[key] = item.scores_delta["seo_delta"] - elif 'best_practices_delta' == json_data[key]: - json_data[key] = item.scores_delta["best_practices_delta"] - elif 'performance_delta' == json_data[key]: - json_data[key] = item.scores_delta["performance_delta"] - elif 'logs' == json_data[key]: - json_data[key] = len(item.logs) - elif 'health' == json_data[key]: - json_data[key] = item.scores["average"] - - get_list.append(f'{key}={json_data[key]}&') - - get_params = ''.join(get_list) - - try: - if request_type == 'POST': - request = requests.post(request_url, data=json_data) - elif request_data == 'GET': - request = requests.get(request_url, params=json_data) - - except: - data = {'success': False} - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - -def automation_phone(phone_number=None, automation_id=None, scan_or_test_id=None): - if phone_number and automation_id and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - account_sid = os.environ.get("TWILIO_SID") - auth_token = os.environ.get("TWILIO_AUTH_TOKEN") - client = Client(account_sid, auth_token) - - message = client.messages.create( - to=phone_number, - from_=os.environ.get('TWILIO_NUMBER'), - body=body - ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - -def automation_slack(automation_id=None, scan_or_test_id=None): - if automation_id and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - account = Account.objects.get(user=automation.user) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - token = account.slack['bot_access_token'] - channel = account.slack['slack_channel_id'] - - client = WebClient(token=token) - try: - response = client.chat_postMessage( - channel=channel, - text=(body), - block=[ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": body, - - } - } - ] - ) - except SlackApiError as e: - assert e.response["error"] - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data \ No newline at end of file diff --git a/app/api/scan_tests/automations.py b/app/api/scan_tests/automations.py deleted file mode 100644 index 6ab60fde..00000000 --- a/app/api/scan_tests/automations.py +++ /dev/null @@ -1,110 +0,0 @@ -from ..models import ( - Automation, Test, Scan, Site, User, - ) -from .alerts import ( - automation_email, automation_webhook, - automation_phone, automation_slack, - ) -import re, uuid - - - - -def automation(automation_id, scan_or_test_id): - automation = Automation.objects.get(id=automation_id) - expressions = automation.expressions - exp_list = [] - actions = automation.actions - act_list = [] - - try: - scan = Scan.objects.get(id=scan_or_test_id) - except: - try: - test = Test.objects.get(id=scan_or_test_id) - except: - return False - - - for expression in expressions: - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' - - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: - joiner = '' - - if 'test_score' in expression['data_type']: - data_type = 'float(test.score)' - elif 'current_average' in expression['data_type']: - data_type = 'float(test.scores_delta["current_average"])' - elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["seo_delta"])' - elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["best_practices_delta"])' - elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["performance_delta"])' - elif 'logs' in expression['data_type']: - data_type = 'len(scan.logs)' - elif 'health' in expression['data_type']: - data_type = 'float(scan.scores["average"])' - - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - exp = f'{joiner}{data_type}{operator}{value}' - exp_list.append(exp) - - - for action in actions: - - if 'slack' in action['action_type']: - action_type = f"\n print('sending slack alert')\ - \n automation_slack(automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'webhook' in action['action_type']: - action_type = f"\n print('sending webhook alert')\ - \n automation_webhook(request_type='{action['request']}', \ - request_url='{action['url']}', request_data='{action['json']}', \ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'email' in action['action_type']: - action_type = f"\n print('sending email alert')\ - \n automation_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'phone' in action['action_type']: - action_type = f"\n print('sending phone alert')\ - \n automation_phone(phone_number='{action['phone']}', \ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - act = f'{action_type}' - act_list.append(act) - - - exp_string = ' '.join(exp_list) - act_string = ''.join(act_list) - - automation_logic = f'if {exp_string}:{act_string}' - exec(automation_logic) - - return True - - - - - - - - - - - - \ No newline at end of file diff --git a/app/api/scan_tests/driver_init.py b/app/api/scan_tests/driver_init.py deleted file mode 100644 index 7f2d38f3..00000000 --- a/app/api/scan_tests/driver_init.py +++ /dev/null @@ -1,35 +0,0 @@ -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -import time, os - - - -def driver_init(): - - prefs = { - 'download.prompt_for_download': False, - 'download.extensions_to_open': '.zip', - 'safebrowsing.enabled': True - } - chrome_path = os.environ.get('CHROMEDRIVER') - WINDOW_SIZE = "1920,1080" - options = webdriver.ChromeOptions() - options.add_experimental_option('prefs',prefs) - options.add_argument("start-maximized") - options.add_argument("--headless") - options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) - options.add_argument('--no-sandbox') - options.add_argument('--disable-dev-shm-usage') - options.add_argument("--disable-extensions") - options.add_argument("--window-size=%s" % WINDOW_SIZE) - options.add_argument("--safebrowsing-disable-download-protection") - options.add_argument("safebrowsing-disable-extension-blacklist") - options.add_argument("--disable-gpu") - - driver = webdriver.Chrome(executable_path=chrome_path, options=options) - driver.set_page_load_timeout(20) - driver.set_script_timeout(20) - driver.implicitly_wait(20) - - - return driver \ No newline at end of file diff --git a/app/api/scan_tests/scan_site.py b/app/api/scan_tests/scan_site.py deleted file mode 100644 index 24711f98..00000000 --- a/app/api/scan_tests/scan_site.py +++ /dev/null @@ -1,100 +0,0 @@ -from .driver_init import driver_init -from selenium import webdriver -from ..models import Site, Scan, Test -from selenium.webdriver.chrome.options import Options -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder -from .lighthouse import Lighthouse -import time, os, sys, json - - - -class ScanSite(): - - def __init__(self, site=None, scan=None): - if site == None and scan != None: - site = scan.site - self.site = site - self.driver = driver_init() - self.scan = scan - - - def first_scan(self): - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - - - if self.scan: - self.scan.html = html - self.scan.logs = logs - self.scan.scores = lh_data["scores"] - self.scan.audits = lh_data["audits"] - self.scan.save() - first_scan = self.scan - else: - first_scan = Scan.objects.create( - site=self.site, html=html, - logs=logs, scores=lh_data["scores"], - audits=lh_data["audits"] - ) - - self.update_site_info(first_scan) - - return first_scan - - - def second_scan(self): - first_scan = Scan.objects.filter( - site=self.site - ).order_by('-time_created').first() - - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - - second_scan = Scan.objects.create( - site=self.site, paired_scan=first_scan, - html=html, logs=logs, scores=lh_data['scores'], - audits=lh_data['audits'] - ) - second_scan.save() - - first_scan.paried_scan = second_scan - first_scan.save() - - self.update_site_info(second_scan) - - return second_scan - - - - def update_site_info(self, scan): - if scan.scores['average'] == None: - health = 'No Data' - badge = 'neutral' - elif float(scan.scores['average']) >= 75: - health = 'Good' - badge = 'success' - elif 75 > float(scan.scores['average']) >= 60: - health = 'Okay' - badge = 'warning' - elif 60 > float(scan.scores['average']): - health = 'Poor' - badge = 'danger' - - self.site.info['latest_scan']['id'] = str(scan.id) - self.site.info['latest_scan']['time_created'] = str(scan.time_created) - self.site.info['lighthouse'] = scan.scores - self.site.info['status']['health'] = str(health) - self.site.info['status']['badge'] = str(badge) - - self.site.save() - - return self.site \ No newline at end of file diff --git a/app/api/tasks.py b/app/api/tasks.py index 190833f5..b20586c3 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -2,7 +2,8 @@ from celery.utils.log import get_task_logger from celery import shared_task from .v1.ops.tasks import (create_site_task, - create_scan_task, create_test_task + create_scan_task, create_test_task, delete_site_s3, + create_report_task, delete_report_s3, ) from .models import Log from django.contrib.auth.models import User @@ -16,17 +17,66 @@ def create_site_bg(site_id): logger.info('Created scan of new site') + @shared_task -def create_scan_bg(site_id, automation_id=None): - create_scan_task(site_id, automation_id) +def create_scan_bg( + scan_id=None, + site_id=None, + automation_id=None, + configs=None, + type=None, + ): + create_scan_task( + scan_id, + site_id, + automation_id, + configs, + type, + ) logger.info('Created new scan of site') + @shared_task -def create_test_bg(site_id, automation_id=None): - create_test_task(site_id, automation_id) +def create_test_bg( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + ): + create_test_task( + test_id, + site_id, + automation_id, + configs, + type, + index, + pre_scan, + post_scan + ) logger.info('Created new test of site') +@shared_task +def create_report_bg(site_id=None, automation_id=None): + create_report_task(site_id, automation_id) + logger.info('Created new report of site') + + +@shared_task +def delete_site_s3_bg(site_id): + delete_site_s3(site_id) + logger.info('Deleted site s3 objects') + + +@shared_task +def delete_report_s3_bg(report_id): + delete_report_s3(report_id) + logger.info('Deleted Report pdf in s3') + @shared_task def purge_logs(username=None): diff --git a/app/api/scan_tests/__init__.py b/app/api/utils/__init__.py similarity index 100% rename from app/api/scan_tests/__init__.py rename to app/api/utils/__init__.py diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py new file mode 100644 index 00000000..1f2d7410 --- /dev/null +++ b/app/api/utils/alerts.py @@ -0,0 +1,519 @@ +from django.core.mail import send_mail, send_mass_mail +from django.contrib.auth.models import User +from django.template.loader import render_to_string +from datetime import date +import os, operator, json, requests, uuid +from django.utils.html import strip_tags +from django.contrib.auth.models import User +from rest_framework.response import Response +from ..models import * +from twilio.rest import Client +from slack_sdk.web import WebClient +from slack_sdk.errors import SlackApiError + + + + + +def create_exp_str(item, automation, is_email=False): + + exp_list = [] + + for e in automation.expressions: + if 'test_score' in e['data_type']: + data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' + elif 'current_health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' + elif 'health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' + + elif 'current_lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' + elif 'seo_delta' in e['data_type']: + data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' + elif 'best_practices_delta' in e['data_type']: + data_type = 'Best Practicies Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' + elif 'performance_delta' in e['data_type']: + data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' + elif 'accessibility_delta' in e['data_type']: + data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' + # LH scan data + elif 'lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + elif 'seo' in e['data_type']: + data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' + elif 'best_practices' in e['data_type']: + data_type = 'Best Practicies:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' + elif 'performance' in e['data_type']: + data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' + elif 'accessibility' in e['data_type']: + data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' + + + + # yellowlab test data + elif 'current_yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' + elif 'pageWeight_delta' in e['data_type']: + data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' + elif 'requests_delta' in e['data_type']: + data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' + elif 'domComplexity_delta' in e['data_type']: + data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' + elif 'javascriptComplexity_delta' in e['data_type']: + data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t' + elif 'badJavascript_delta' in e['data_type']: + data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t' + elif 'jQuery_delta' in e['data_type']: + data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t' + elif 'cssComplexity_delta' in e['data_type']: + data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t' + elif 'badCSS_delta' in e['data_type']: + data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t' + elif 'fonts_delta' in e['data_type']: + data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t' + elif 'serverConfig_delta' in e['data_type']: + data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t' + + # yellowlab scan data + elif 'yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' + elif 'pageWeight' in e['data_type']: + data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' + elif 'requests' in e['data_type']: + data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' + elif 'domComplexity' in e['data_type']: + data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' + elif 'javascriptComplexity' in e['data_type']: + data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t' + elif 'badJavascript' in e['data_type']: + data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t' + elif 'jQuery' in e['data_type']: + data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t' + elif 'cssComplexity' in e['data_type']: + data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t' + elif 'badCSS' in e['data_type']: + data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t' + elif 'fonts' in e['data_type']: + data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t' + elif 'serverConfig' in e['data_type']: + data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' + + elif 'images_score' in e['data_type']: + data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' + elif 'logs' in e['data_type']: + data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + + + exp_list.append(data_type) + + if is_email: + return exp_list + + exp_str = ('\t'+''.join(exp_list)) + return exp_str + + + + + + + + + + + +def create_json_data(data, obj): + json_data = data + item = obj + + for key in json_data: + if 'test_score' == json_data[key]: + json_data[key] = item.score + elif 'seo_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] + elif 'best_practices_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] + elif 'performance_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] + elif 'accessibility_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] + elif 'current_health' == json_data[key]: + json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) + elif 'health' == json_data[key]: + json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) + elif 'logs' == json_data[key]: + json_data[key] = len(item.logs) + elif 'current_lighthouse_average' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["current_average"] + elif 'current_average' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["current_average"] + elif 'seo' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["seo"] + elif 'best_practice' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["best_practices"] + elif 'performance' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["performance"] + elif 'accessibility' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["accessibility"] + + elif 'current_yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["current_average"] + elif 'pageWeight_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] + elif 'requests_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] + elif 'domComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] + elif 'javascriptComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"] + elif 'badJavascript_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"] + elif 'jQuery_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"] + elif 'cssComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"] + elif 'badCSS_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"] + elif 'fonts_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"] + elif 'serverConfig_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] + + elif 'yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["globalScore"] + elif 'pageWeight' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["pageWeight"] + elif 'requests' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["requests"] + elif 'domComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["domComplexity"] + elif 'javascriptComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["javascriptComplexity"] + elif 'badJavascript' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badJavascript"] + elif 'jQuery' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["jQuery"] + elif 'cssComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["cssComplexity"] + elif 'badCSS' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badCSS"] + elif 'fonts' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["fonts"] + elif 'serverConfig' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["serverConfig"] + + + return json_data + + + + + + + + + + + + +def automation_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_list = create_exp_str(item=item, automation=automation, is_email=True) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + subject = f'Alert for {site.site_url}' + title = f'Alert for {site.site_url}' + pre_header = f'Alert for {site.site_url}' + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Site Dashboard', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_report_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Report.objects.get(id=uuid.UUID(object_id)) + item_type = 'Report' + except: + return {'success': False} + + exp_list = '' + object_url = str(item.path) + subject = f'Report for {site.site_url}' + title = f'Report for {site.site_url}' + pre_header = f'Report for {site.site_url}' + pre_content = ( + f'Scanerr just finished creating a {item_type} for {site.site_url}. ' + f'Please click the link below to access and download the report.\n' + ) + content = ( + f'This message was triggered by an automation created with Scanerr. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Report', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_webhook( + request_type=None, + request_url=None, + request_data=None, + automation_id=None, + object_id=None, + ): + if request_type and automation_id and request_url and request_data and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + pre_json_data = json.loads(request_data) + json_data = create_json_data(data=pre_json_data, obj=item) + + try: + if request_type == 'POST': + response = requests.post(request_url, data=json_data) + elif request_data == 'GET': + response = requests.get(request_url, params=json_data) + + print(response.json()) + + except: + data = {'success': False} + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_phone(phone_number=None, automation_id=None, object_id=None): + if phone_number and automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + account_sid = os.environ.get("TWILIO_SID") + auth_token = os.environ.get("TWILIO_AUTH_TOKEN") + client = Client(account_sid, auth_token) + + message = client.messages.create( + to=phone_number, + from_=os.environ.get('TWILIO_NUMBER'), + body=body + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_slack(automation_id=None, object_id=None): + if automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + account = Account.objects.get(user=automation.user) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + token = account.slack['bot_access_token'] + channel = account.slack['slack_channel_id'] + + client = WebClient(token=token) + try: + response = client.chat_postMessage( + channel=channel, + text=(body), + block=[ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": body, + + } + } + ] + ) + except SlackApiError as e: + assert e.response["error"] + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data \ No newline at end of file diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py new file mode 100644 index 00000000..4d93eb96 --- /dev/null +++ b/app/api/utils/automations.py @@ -0,0 +1,206 @@ +from ..models import * +from .alerts import * +import re, uuid + + + +def automation(automation_id, object_id): + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + expressions = automation.expressions + exp_list = [] + actions = automation.actions + act_list = [] + scan = None + test = None + report = None + use_exp = True + + if schedule.task_type == 'scan': + try: + scan = Scan.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'test': + try: + test = Test.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'report': + try: + report = Report.objects.get(id=object_id) + use_exp = False + except: + return False + else: + return False + + + + if use_exp: + for expression in expressions: + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + if 'test_score' in expression['data_type']: + data_type = 'float(test.score)' + + # lighthouse test data + elif 'current_lighthouse_average' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' + elif 'seo_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' + elif 'best_practices_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' + elif 'performance_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' + elif 'accessibility_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' + # lighthouse scan data + elif 'lighthouse_average' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["average"])' + elif 'seo' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["seo"])' + elif 'best_practices' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["best_practices"])' + elif 'performance' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["performance"])' + elif 'accessibility' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["accessibility"])' + + # yellowlab test data + elif 'current_yellowlab_average' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' + elif 'pageWeight_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' + elif 'requests_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' + elif 'domComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' + elif 'javascriptComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' + elif 'badJavascript_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' + elif 'jQuery_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' + elif 'cssComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' + elif 'badCSS_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' + elif 'fonts_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' + elif 'serverConfig_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' + # yellowlab scan data + elif 'yellowlab_average' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["globalScore"])' + elif 'pageWeight' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' + elif 'requests' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["requests"])' + elif 'domComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' + elif 'javascriptComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' + elif 'badJavascript' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' + elif 'jQuery' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["jQuery"])' + elif 'cssComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' + elif 'badCSS' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badCSS"])' + elif 'fonts' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["fonts"])' + elif 'serverConfig' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' + + + elif 'logs' in expression['data_type']: + data_type = 'len(scan.logs)' + + elif 'current_health' in expression['data_type']: + data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' + + elif 'health' in expression['data_type']: + data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' + + elif 'images_score' in expression['data_type']: + data_type = 'float(test.images_delta["average_score"])' + + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + exp = f'{joiner}{data_type}{operator}{value}' + exp_list.append(exp) + + + + for action in actions: + + if 'slack' in action['action_type']: + action_type = f"\n print('sending slack alert')\ + \n automation_slack(automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'webhook' in action['action_type']: + action_type = f"\n print('sending webhook alert')\ + \n automation_webhook(request_type='{action['request']}', \ + request_url='{action['url']}', request_data='{action['json']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'email' in action['action_type']: + action_type = f"\n print('sending email alert')\ + \n automation_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if report: + action_type = f"\n print('sending report email')\ + \n automation_report_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'phone' in action['action_type']: + action_type = f"\n print('sending phone alert')\ + \n automation_phone(phone_number='{action['phone']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + act = f'{action_type}' + act_list.append(act) + + + exp_string = ' '.join(exp_list) + act_string = ''.join(act_list) + + if not use_exp: + exp_string = '1 == 1' + + automation_logic = f'if {exp_string}:{act_string}' + print(automation_logic) + exec(automation_logic) + + return True + + + + + + + + + + + + \ No newline at end of file diff --git a/app/api/scan_tests/custom-config.js b/app/api/utils/custom-config.js similarity index 100% rename from app/api/scan_tests/custom-config.js rename to app/api/utils/custom-config.js diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py new file mode 100644 index 00000000..3886db01 --- /dev/null +++ b/app/api/utils/driver.py @@ -0,0 +1,107 @@ +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver import ActionChains +import time, os, numpy, json + + + +def driver_init( + window_size='1920,1080', + script_timeout=30, + load_timeout=30, + wait_time=15, + ): + + prefs = { + 'download.prompt_for_download': False, + 'download.extensions_to_open': '.zip', + 'safebrowsing.enabled': True + } + chrome_path = os.environ.get("CHROMEDRIVER") + options = webdriver.ChromeOptions() + options.add_experimental_option('prefs',prefs) + options.add_argument("start-maximized") + options.add_argument("--headless") + options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-extensions") + options.add_argument("--window-size=%s" % window_size) + options.add_argument("--safebrowsing-disable-download-protection") + options.add_argument("safebrowsing-disable-extension-blacklist") + options.add_argument("--disable-gpu") + + caps = DesiredCapabilities.CHROME + #as per latest docs + caps['goog:loggingPrefs'] = {'performance': 'ALL'} + + driver = webdriver.Chrome(executable_path=chrome_path, options=options, desired_capabilities=caps) + driver.set_page_load_timeout(load_timeout) + driver.set_script_timeout(script_timeout) + driver.implicitly_wait(wait_time) + + + return driver + + + + +def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): + """ + Pauses the driver until all network requests have been resolved + + --> Adding mouse interaction to load WP plugin rendered content + + returns once driver determines that all request have resolved or + total wait time exceeds max_wait_time + + """ + + def get_request_list(driver): + # get current snapshot of driver requests + requests = driver.get_log('performance') + r_list = [] + for r in requests: + network_log = json.loads(r["message"])["message"] + + # Checks if the current 'method' key has any Network related value. + if("Network.response" in network_log["method"] + or "Network.request" in network_log["method"] + or "Network.webSocket" in network_log["method"]): + + r_list.append(network_log) + + return r_list + + + def interact_with_page(driver): + # simulate mouse movement and click on tag + html_tag = driver.find_elements_by_tag_name('html')[0] + action = ActionChains(driver) + action.move_to_element(html_tag).perform() + return + + + resolved = False + wait_time = 0 + + # actions before comparing network logs + interact_with_page(driver) + time.sleep(min_wait_time) + + while not resolved and wait_time < max_wait_time: + # get first set of logs + list_one = get_request_list(driver=driver) + + # wait 5 sec or sec for request to resolve + time.sleep(interval) + + # get second set of logs + list_two = get_request_list(driver=driver) + + # check if logs are equal + resolved = numpy.array_equal(list_one, list_two) + + wait_time += interval + + return \ No newline at end of file diff --git a/app/api/utils/image.py b/app/api/utils/image.py new file mode 100644 index 00000000..11c89636 --- /dev/null +++ b/app/api/utils/image.py @@ -0,0 +1,299 @@ +from .driver import driver_init, driver_wait +from selenium import webdriver +from ..models import Site, Scan, Test +from selenium.webdriver.chrome.options import Options +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from sewar.full_ref import uqi, mse, ssim +from scanerr import settings +from PIL import Image as I +import time, os, sys, json, uuid, boto3, statistics, shutil, numpy + + + + +class Image(): + """ + High level Image handler used to compare screenshots of + a website. Also known as VRT or Visual Regression Testing. + Contains two methods scan() and test(): + + def scan(site, driver=None) -> grabs multiple + screenshots of the website and uploads + them to s3. + + + def test(test=) -> compares each + screenshot in the two scans and records + a score out of 100% + + + def screeshot(site, driver=None) -> grabs single + screenshot of the site and uploads it to s3 + + """ + + + + def scan(self, site, configs, driver=None,): + """ + Grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + driver.execute_script("window.scrollBy(0, window.innerHeight);") + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + window.innerHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + if not driver_present: + driver.quit() + + return image_array + + + + + + + + def test(self, test, index=None): + """ + compares each screenshot between the two scans and records + a score out of 100%. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setup temp files + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.site.id}') + + # loop through and download each img in scan and compare it. + pre_scan_images = test.pre_scan.images + img_test_results = [] + scores = [] + i = 0 + + if index is not None: + pre_scan_images = [test.pre_scan.images[index]] + i = index + + for pre_img_obj in pre_scan_images: + + # getting pre_scan image + pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') + with open(pre_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + + # open with PIL Image library + pre_img = I.open(pre_img_path) + # convert to array + pre_img_array = numpy.array(pre_img) + + # getting post_scan image + try: + post_img_obj = test.post_scan.images[i] + except: + post_img_obj = None + + if post_img_obj is not None: + post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') + with open(post_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + + # open with PIL Image library + post_img = I.open(post_img_path) + # convert to array + post_img_array = numpy.array(post_img) + + # test images + img_score_tupple = ssim(pre_img_array, post_img_array) + img_score_list = list(img_score_tupple) + img_score = statistics.fmean(img_score_list) * 100 + + # create img test obj and add to array + img_test_obj = { + "index": i, + "pre_img": pre_img_obj, + "post_img": post_img_obj, + "score": img_score, + } + + img_test_results.append(img_test_obj) + scores.append(img_score) + + # remove local copies + if post_img_obj is not None: + os.remove(post_img_path) + os.remove(pre_img_path) + + i += 1 + + # remove temp dir + shutil.rmtree(temp_root) + + # averaging scores and storing in images_delta obj + avg_score = statistics.fmean(scores) + images_delta = { + "average_score": avg_score, + "images": img_test_results, + } + + return images_delta + + + + + + + + def screenshot(self, site=None, url=None, configs=None, driver=None,): + """ + Grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10 + } + + # initialize driver if not passed as param + if not driver: + driver = driver_init(window_size=configs['window_size']) + + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + driver.get(site_url) + + + # wait for site to fully load + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # grab screenshot + pic_id = uuid.uuid4() + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + return img_obj \ No newline at end of file diff --git a/app/api/scan_tests/lighthouse.py b/app/api/utils/lighthouse.py similarity index 80% rename from app/api/scan_tests/lighthouse.py rename to app/api/utils/lighthouse.py index 4a20400e..af2207f8 100644 --- a/app/api/scan_tests/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -1,13 +1,11 @@ -from io import StringIO -import os, fileinput, glob, subprocess, time, sys, json +import subprocess, json from ..models import Site, Scan -from django.forms.models import model_to_dict class Lighthouse(): - """Initialized Google's Lighthouse CLI and runs an audit of the site""" + """Initializes Google's Lighthouse CLI and runs an audit of the site""" def __init__(self, site=None): @@ -17,7 +15,7 @@ def __init__(self, site=None): def init_audit(self): proc = subprocess.Popen([ 'lighthouse', - '--config-path=api/scan_tests/custom-config.js', + '--config-path=api/utils/custom-config.js', '--quiet', self.site.site_url, '--chrome-flags="--no-sandbox --headless"', @@ -25,6 +23,7 @@ def init_audit(self): 'json', ], stdout=subprocess.PIPE, + user='app', ) stdout_value = proc.communicate()[0] return stdout_value @@ -51,14 +50,15 @@ def get_data(self): "best-practices": [], } - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` list + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj for cat in audits: cat_audits = stdout_json["categories"][cat]["auditRefs"] for a in cat_audits: if int(a["weight"]) > 0: audit = stdout_json["audits"][a["id"]] audits[cat].append(audit) - + # changing audits name of best-practices to best_practices + audits['best_practices'] = audits.pop('best-practices') # get scores from each category seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) @@ -68,13 +68,14 @@ def get_data(self): average_score = (seo_score + accessibility_score + performance_score + best_practices_score)/4 scores = { - "seo": str(seo_score), - "accessibility": str(accessibility_score), - "performance": str(performance_score), - "best_practices": str(best_practices_score), - "average": str(average_score), + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "average": average_score, } + data = { "scores": scores, "audits": audits @@ -96,7 +97,7 @@ def get_data(self): "seo": [], "accessibility": [], "performance": [], - "best_practices": [], + "best-practices": [], } data = { diff --git a/app/api/utils/report_assets/cover_img.png b/app/api/utils/report_assets/cover_img.png new file mode 100644 index 00000000..ac6e1b5e Binary files /dev/null and b/app/api/utils/report_assets/cover_img.png differ diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py new file mode 100644 index 00000000..bf741fb4 --- /dev/null +++ b/app/api/utils/reporter.py @@ -0,0 +1,459 @@ +from ..models import * +import time, os, sys, json, boto3 +import PIL.Image as Img +from scanerr import settings +from datetime import datetime, timedelta +from reportlab.lib.pagesizes import letter +from reportlab.lib.units import inch +from reportlab.lib.colors import HexColor +from reportlab.pdfgen import canvas + + + +class Reporter(): + + ''' + Used for generating web vitals reports for the passed `Site` obj + + Expects -> { + "report": , + } + + returns --> + + ''' + + def __init__(self, report, scan=None): + self.report = report + self.site = self.report.site + if scan is None: + self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) + else: + self.scan = scan + + #building paths & canvas template + if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + else: + os.makedirs(f'{settings.BASE_DIR}/temp') + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + + self.page_index = 0 + self.text_color = self.report.info['text_color'] + self.highlight_color = self.report.info['highlight_color'] + self.background_color = self.report.info['background_color'] + self.c = canvas.Canvas(self.local_path, letter) + self.y = 9 + + + def setup_page(self): + # sets the defaults for a new page + self.c.setFillColor(HexColor(self.background_color)) + self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + + + def end_page(self): + # adds page number and ends page + self.c.setFont('Helvetica-Bold', 15) + self.c.setFillColor(HexColor(self.text_color)) + self.page_index += 1 + self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.showPage() + + + def draw_page_title(self, title): + # adds a title to the given page + self.c.setFont('Helvetica-Bold', 32) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawCentredString(4.25*inch, 10*inch, title) + + + def publish_report(self): + self.c.save() + remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' + s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # uploading package to remote s3 + with open(self.local_path, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + + + + def cover_page(self): + # background and title + self.setup_page() + + # creating dark triangle + p = self.c.beginPath() + p.moveTo(0*inch, 11*inch) + p.lineTo(7*inch, 11*inch) + p.lineTo(2.5*inch, 4.5*inch) + p.lineTo(0*inch, 7*inch) + self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # crating light triangle + p = self.c.beginPath() + p.moveTo(0*inch, 0*inch) + p.lineTo(0*inch, 7*inch) + p.lineTo(7*inch, 0*inch) + self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # date + date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' + self.c.setFont('Helvetica-Bold', 24) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 7.5*inch, date) + + # title + self.c.setFont('Helvetica-Bold', 45) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') + if len(self.site.site_url) <= 15: + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + elif 15 < len(self.site.site_url): + extra_chars = len(self.site.site_url) - 15 + m = (2/5) + self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + # cover img + cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") + self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + + self.end_page() + + + + + def get_score_data(self, score, is_binary=False): + score = float(score) + if is_binary: + score = score*100 + + score_types = { + "a": { + "grade": "A", + "color": "#38B43F", + }, + "b": { + "grade": "B", + "color": "#82B436", + }, + "c": { + "grade": "C", + "color": "#ACB43C", + }, + "d": { + "grade": "D", + "color": "#B49836", + }, + "e": { + "grade": "E", + "color": "#B46B34", + }, + "f": { + "grade": "F", + "color": "#B43A29", + }, + + } + + if score > 80: + grade = score_types['a'] + elif 80 > score > 70: + grade = score_types['b'] + elif 70 > score > 50: + grade = score_types['c'] + elif 50 > score > 30: + grade = score_types['d'] + elif 30 > score > 0: + grade = score_types['e'] + else: + grade = score_types['f'] + + return grade + + + def get_cat_string(self, cat): + + if cat == 'fonts': + string = 'Fonts' + elif cat == 'badCSS': + string = 'Bad CSS' + elif cat == 'jQuery': + string = 'jQuery' + elif cat == 'requests': + string = 'Requests' + elif cat == 'pageWeight': + string = 'Page Weight' + elif cat == 'serverConfig': + string = 'Server Config' + elif cat == 'badJavascript': + string = 'Bad JS' + elif cat == 'cssComplexity': + string = 'CSS Complexity' + elif cat == 'domComplexity': + string = 'DOM Complexity' + elif cat == 'javascriptComplexity': + string = 'JS Complexity' + elif cat == 'seo': + string = 'SEO' + elif cat == 'best_practices' or cat == 'best-practices': + string = 'Best Practices' + elif cat == 'performance': + string = 'Performance' + elif cat == 'accessibility': + string = 'Accessibility' + + return string + + + + def create_data(self, data_type=str): + self.setup_page() + + if data_type == 'yellowlab': + data = self.scan.yellowlab + page_title = 'Yellow Lab' + avg_score = 'globalScore' + + if data_type == 'lighthouse': + data = self.scan.lighthouse + page_title = 'Lighthouse' + avg_score = 'average' + + self.draw_page_title(page_title) + if data['scores'][avg_score] is None: + return False + + # measurements + space = .25 + text_space = .05 + begin_y = 8 + log_margin = 3.7 + text_margin = .3 + value_margin = 3 + log_height = .2 + log_width = 4 + grade_tab_width = .07 + + c_count = 0 + logs_count = 0 + for cat in data['audits']: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data(data['scores'][avg_score]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + + + # creating individual grade cards + grade_obj = self.get_score_data(data['scores'][cat]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 16) + self.c.drawCentredString( + .75*inch, + (begin_y - .07)*inch, + grade_obj['grade'] + ) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) + self.c.drawCentredString( + 2.3*inch, + (begin_y - .07)*inch, + cat_string + ) + + + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data(policy['score'], is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) + + c_count += 1 + + + + self.end_page() + + + + + + + + + + + + + + + + + + + + + + + + + + + + def make_test_report(self): + + self.cover_page() + + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') + + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + if 'crux' in self.report.type or 'full' in self.report.type: + self.setup_page() + self.draw_page_title('CRUX') + self.end_page() + + self.publish_report() + return self.report + + + + + + + \ No newline at end of file diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py new file mode 100644 index 00000000..d5871295 --- /dev/null +++ b/app/api/utils/scanner.py @@ -0,0 +1,161 @@ +from .driver import driver_init +from ..models import Site, Scan, Test +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from .lighthouse import Lighthouse +from .yellowlab import Yellowlab +from .image import Image +import time, os, sys, json + + + +class Scanner(): + + def __init__( + self, + site=None, + scan=None, + configs=None, + ): + + if site == None and scan != None: + site = scan.site + if configs is None: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + self.site = site + self.driver = driver_init(window_size=configs['window_size']) + self.scan = scan + self.configs = configs + + + + def first_scan(self): + """ + Method to run a scan independently of an existing `scan` obj. + + returns -> `Scan` + """ + self.driver.get(self.site.site_url) + time.sleep(5) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + self.driver.quit() + lh_data = Lighthouse(self.site).get_data() + yl_data = Yellowlab(self.site).get_data() + + + if self.scan: + self.scan.html = html + self.scan.logs = logs + self.scan.images = images + self.scan.lighthouse = lh_data + self.scan.yellowlab = yl_data + self.scan.configs = self.configs + self.scan.save() + first_scan = self.scan + else: + first_scan = Scan.objects.create( + site=self.site, html=html, + logs=logs, lighthouse=lh_data, + images=images, yellowlab=yl_data, + configs=self.configs + ) + + self.update_site_info(first_scan) + + return first_scan + + + + + + def second_scan(self): + """ + Method to run a scan and attach existing `scan` obj to it. + + returns -> `Scan` + """ + if not self.scan: + first_scan = Scan.objects.filter( + site=self.site + ).order_by('-time_created').first() + + else: + first_scan = self.scan + + self.driver.get(self.site.site_url) + time.sleep(5) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + self.driver.quit() + lh_data = Lighthouse(self.site).get_data() + yl_data = Yellowlab(self.site).get_data() + + second_scan = Scan.objects.create( + site=self.site, paired_scan=first_scan, + html=html, logs=logs, lighthouse=lh_data, + images=images, yellowlab=yl_data, + configs=self.configs + ) + second_scan.save() + + first_scan.paried_scan = second_scan + first_scan.save() + + self.update_site_info(second_scan) + + return second_scan + + + + def update_site_info(self, scan): + + health = 'No Data' + badge = 'neutral' + d = 0 + score = 0 + + if scan.lighthouse['scores']['average'] is not None: + score += float(scan.lighthouse['scores']['average']) + d += 1 + if scan.yellowlab['scores']['globalScore'] is not None: + score += float(scan.yellowlab['scores']['globalScore']) + d += 1 + + if score != 0: + score = score / d + + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + + else: + if self.site.info['status']['score'] is not None: + score = float(self.site.info['status']['score']) + else: + score = None + + self.site.info['latest_scan']['id'] = str(scan.id) + self.site.info['latest_scan']['time_created'] = str(scan.time_created) + self.site.info['lighthouse'] = scan.lighthouse['scores'] + self.site.info['yellowlab'] = scan.yellowlab['scores'] + self.site.info['status']['health'] = str(health) + self.site.info['status']['badge'] = str(badge) + self.site.info['status']['score'] = score + + self.site.save() + + return self.site \ No newline at end of file diff --git a/app/api/scan_tests/tester.py b/app/api/utils/tester.py similarity index 50% rename from app/api/scan_tests/tester.py rename to app/api/utils/tester.py index d0d45ebc..c3c46b42 100644 --- a/app/api/scan_tests/tester.py +++ b/app/api/utils/tester.py @@ -1,11 +1,12 @@ from ..models import Site, Scan, Test import time, os, sys, json, random, string, re -from difflib import SequenceMatcher, HtmlDiff +from difflib import SequenceMatcher, HtmlDiff, Differ from datetime import datetime +from .image import Image -class Test(): +class Tester(): def __init__(self, test): self.test = test @@ -98,6 +99,7 @@ def compare_html(self): return html_raw_score + def compare_logs(self): self.clean_logs() pre_scan = list(self.pre_scan_logs) @@ -235,16 +237,18 @@ def delta_logs(self): - def delta_scores(self): + + + def delta_lighthouse(self): try: - pre_seo = int(self.test.pre_scan.scores['seo']) - pre_accessibility = int(self.test.pre_scan.scores['accessibility']) - pre_performance = int(self.test.pre_scan.scores['performance']) - pre_best_practices = int(self.test.pre_scan.scores['best_practices']) - post_seo = int(self.test.post_scan.scores['seo']) - post_accessibility = int(self.test.post_scan.scores['accessibility']) - post_performance = int(self.test.post_scan.scores['performance']) - post_best_practices = int(self.test.post_scan.scores['best_practices']) + pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) + pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) + pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) + pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) + post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) + post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) + post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) + post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) seo_delta = post_seo - pre_seo accessibility_delta = post_accessibility - pre_accessibility @@ -252,22 +256,100 @@ def delta_scores(self): best_practices_delta = post_best_practices - pre_best_practices current_average = (post_seo + post_accessibility + post_best_practices + post_performance)/4 old_average = (pre_seo + pre_accessibility + pre_best_practices + pre_performance)/4 - average_diff = current_average - old_average + average_delta = current_average - old_average except: seo_delta = None accessibility_delta = None performance_delta = None best_practices_delta = None current_average = None - average_diff = None + average_delta = None + + data = { + "scores": { + "seo_delta": seo_delta, + "accessibility_delta": accessibility_delta, + "performance_delta": performance_delta, + "best_practices_delta": best_practices_delta, + "current_average": current_average, + "average_delta": average_delta, + } + } + + return data + + + + + + + def delta_yellowlab(self): + try: + pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) + pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) + pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) + pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) + pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) + pre_jQuery = int(self.test.pre_scan.yellowlab["scores"]['jQuery']) + pre_cssComplexity = int(self.test.pre_scan.yellowlab["scores"]['cssComplexity']) + pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) + pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) + pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) + + post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) + post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) + post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) + post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) + post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) + post_jQuery = int(self.test.post_scan.yellowlab["scores"]['jQuery']) + post_cssComplexity = int(self.test.post_scan.yellowlab["scores"]['cssComplexity']) + post_badCSS = int(self.test.post_scan.yellowlab["scores"]['badCSS']) + post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) + post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + + pageWeight_delta = post_pageWeight - pre_pageWeight + requests_delta = post_requests - pre_requests + domComplexity_delta = post_domComplexity - pre_domComplexity + javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity + badJavascript_delta = post_badJavascript - pre_badJavascript + jQuery_delta = post_jQuery - pre_jQuery + cssComplexity_delta = post_cssComplexity - pre_cssComplexity + badCSS_delta = post_badCSS - pre_badCSS + fonts_delta = post_fonts - pre_fonts + serverConfig_delta = post_serverConfig - pre_serverConfig + + average_delta = post_globalScore - pre_globalScore + + except: + pageWeight_delta = None + requests_delta = None + domComplexity_delta = None + javascriptComplexity_delta = None + badJavascript_delta = None + jQuery_delta = None + cssComplexity_delta = None + badCSS_delta = None + fonts_delta = None + serverConfig_delta = None + average_delta = None data = { - "seo_delta": seo_delta, - "accessibility_delta": accessibility_delta, - "performance_delta": performance_delta, - "best_practices_delta": best_practices_delta, - "current_average": current_average, - "average_diff": average_diff, + "scores": { + "pageWeight_delta": pageWeight_delta, + "requests_delta": requests_delta, + "domComplexity_delta": domComplexity_delta, + "javascriptComplexity_delta": javascriptComplexity_delta, + "badJavascript_delta": badJavascript_delta, + "jQuery_delta": jQuery_delta, + "cssComplexity_delta": cssComplexity_delta, + "badCSS_delta": badCSS_delta, + "fonts_delta": fonts_delta, + "serverConfig_delta": serverConfig_delta, + "average_delta": average_delta, + "current_average": post_globalScore, + } } return data @@ -278,7 +360,7 @@ def update_site_info(self, test): site = test.site site.info['latest_test']['id'] = str(test.id) site.info['latest_test']['time_created'] = str(test.time_created) - site.info['latest_test']['score'] = str(round(test.score)) + site.info['latest_test']['score'] = (round(test.score * 100) / 100) site.save() return site @@ -286,36 +368,134 @@ def update_site_info(self, test): - def run_full_test(self): - html_score = self.compare_html() - logs_score = self.compare_logs() - delta_html_data = self.delta_html() - delta_logs_data = self.delta_logs() - delta_scores_data = self.delta_scores() - num_html_ratio = delta_html_data['num_html_ratio'] - num_logs_ratio = delta_logs_data['num_logs_ratio'] - delta_scores_avg_diff = delta_scores_data['average_diff'] - if delta_scores_avg_diff != None: - delta_scores = (100 + delta_scores_avg_diff)/100 - else: - delta_scores = 0 - micro_diff_score = self.html_micro_diff_score( - delta_html_data['post_micro_delta']['delta_parsed_diff'] - ) - html_score_w = 1 - logs_score_w = 1 - num_logs_w = 2 - num_html_w = 1 - micro_diff_w = 2 + + + + def run_test(self, index=None): + + # default scores + html_score = 0 + num_html_ratio = 0 + micro_diff_score = 0 + logs_score = 0 + num_logs_ratio = 0 + lighthouse_score = 0 + yellowlab_score = 0 + images_score = 0 + + # default weights + html_score_w = 0 + num_html_w = 0 + micro_diff_w = 0 + logs_score_w = 0 + num_logs_w = 0 + delta_lh_w = 0 + delta_yl_w = 0 + images_w = 0 + + # default data + html_delta_context = None + logs_delta_context = None + lighthouse_data = None + yellowlab_data = None + images_data = None + + + + if 'html' in self.test.type or 'full' in self.test.type: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( + delta_html_data['post_micro_delta']['delta_parsed_diff'] + ) + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + + + if 'logs' in self.test.type or 'full' in self.test.type: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + } + + + + if 'lighthouse' in self.test.type or 'full' in self.test.type: + # scores & data + lighthouse_data = self.delta_lighthouse() + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None: + lighthouse_score = (100 + lighthouse_avg)/100 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 0: + delta_lh_w = 1 + lighthouse_score = 1 + else: + delta_lh_w = 1 + + + + + if 'yellowlab' in self.test.type or 'full' in self.test.type: + # scores & data + yellowlab_data = self.delta_yellowlab() + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None: + yellowlab_score = (100 + yellowlab_avg)/100 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 0: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + + + + + if 'vrt' in self.test.type or 'full' in self.test.type: + # scores & data + images_data = Image().test(test=self.test, index=index) + images_score = images_data['average_score'] / 100 + + # weights + images_w = 2 + - if delta_scores_avg_diff > 0 or delta_scores_avg_diff == None: - delta_scores_w = 0 - else: - delta_scores_w = 1 total_w = ( html_score_w + logs_score_w + num_html_w - + num_logs_w + delta_scores_w + micro_diff_w + + num_logs_w + delta_lh_w + micro_diff_w + + images_w + delta_yl_w ) @@ -324,34 +504,30 @@ def run_full_test(self): (logs_score * logs_score_w) + (num_logs_ratio * num_logs_w) + (num_html_ratio * num_html_w) + - (delta_scores * delta_scores_w) + - (micro_diff_score * micro_diff_w) + (lighthouse_score * delta_lh_w) + + (yellowlab_score * delta_yl_w) + + (micro_diff_score * micro_diff_w) + + (images_score * images_w) ) / total_w) * 100 + print( "Formula was --> ((" + str(html_score*html_score_w) + " + " + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " - + str(num_html_ratio*num_html_w) + " + " + str(delta_scores*delta_scores_w) + - " + " + str(micro_diff_score*micro_diff_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) - ) - - html_delta_context = { - "pre_html_delta": delta_html_data['delta_html_pre'], - "post_html_delta": delta_html_data['delta_html_post'], - "pre_micro_delta": delta_html_data['pre_micro_delta'], - "post_micro_delta": delta_html_data['post_micro_delta'], - } + + str(num_html_ratio*num_html_w) + " + " + str(lighthouse_score*delta_lh_w) + + " + " + str(micro_diff_score*micro_diff_w) + " + " + str(images_score * images_w)+ + " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) + ) - logs_delta_context = { - "pre_logs_delta": delta_logs_data['delta_logs_pre'], - "post_logs_delta": delta_logs_data['delta_logs_post'], - } self.test.time_completed = datetime.now() self.test.html_delta = html_delta_context self.test.logs_delta = logs_delta_context + self.test.lighthouse_delta = lighthouse_data + self.test.yellowlab_delta = yellowlab_data + self.test.images_delta = images_data self.test.score = score - self.test.scores_delta = delta_scores_data + self.test.save() self.update_site_info(self.test) diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py new file mode 100644 index 00000000..39a826f1 --- /dev/null +++ b/app/api/utils/wordpress.py @@ -0,0 +1,350 @@ +from .driver import driver_init, driver_wait +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + if wait_time is None: + self.driver = driver_init() + else: + self.driver = driver_init(wait_time=wait_time) + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + + + + def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + try: + self.driver.get(self.login_url) + try: + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_xpath( + '//*[@id="jetpack-sso-wrap"]/a[1]').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_link_text( + 'Login with username and password').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + + + try: + verify_email = self.driver.find_element_by_xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + self.driver.execute_script('arguments[0].click();', verify_email) + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error') + self.driver.refresh() + + print('trying login again') + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + self.driver.quit() + return False + + + # removing alerts + try: + deny_btn = self.driver.find_element_by_id('webpushr-deny-button') + self.driver.execute_script("arguments[0].click();", deny_btn) + print('removed alert') + except: + pass + + try: + # checking if url location is wp-admin + current_url = str(self.driver.current_url) + admin_link = '/wp-admin/' + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + pass + else: + print('not in wp-admin - navigating there now') + admin_btn = self.driver.find_element_by_id('wp-admin-bar-dashboard') + admin_link = admin_btn.find_element_by_tag_name('a') + self.driver.execute_script("arguments[0].click();", admin_link) + print('clicked dashboard link') + + except: + print('could not login') + self.driver.quit() + return False + + + return True + + + + + + def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + s_url) + + # finding and recording current native language + lang_selector = self.driver.find_element_by_id('WPLANG') + optgroup = lang_selector.find_elements_by_tag_name('optgroup')[0] + selected_lang = optgroup.find_element_by_xpath('.//option[@selected="selected"]') + default_lang = selected_lang.get_attribute('lang') + default_lang_value = selected_lang.get_attribute('value') + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + select = Select(lang_selector) + select.select_by_value('en_CA') + print('selected english') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + '/' + s_url) + + # selecting native lang + lang_selector = self.driver.find_element_by_id('WPLANG') + select = Select(lang_selector) + select.select_by_value(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved native lang') + + except: + self.driver.quit() + return False + + self.driver.quit() + return True + + + + def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = self.driver.find_element_by_xpath('//*[@id="menu-plugins"]') + self.driver.execute_script("arguments[0].click();", plugin_menu) + p_url = 'plugins.php' + plugins = self.driver.find_element_by_xpath('.//a[@href="'+p_url+'"]') + self.driver.execute_script("arguments[0].click();", plugins) + print('clicked plugin menu') + + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + + except: + print('trying link method for navigation') + try: + self.driver.get(self.admin_link + plugin_menu_page) + time.sleep(10) + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + except: + print('unable to find plugin table') + self.driver.quit() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = self.driver.find_element_by_xpath('//a[@href="'+url+'"]') + self.driver.execute_script("arguments[0].click();", add_plugin) + print('clicked add plugin link') + time.sleep(5) + except: + self.driver.get(self.admin_url + add_plugin_page) + time.sleep(5) + + + # searching for plugin + search_form = self.driver.find_element_by_xpath('//input[@type="search"]') + search_form.clear() + search_form.send_keys(plugin_name) + time.sleep(1) + search_form.send_keys(Keys.RETURN) + time.sleep(3) + + ##### Clicking Updraft "install" plugin ###### + install = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", install) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', install) + print('Clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + self.driver.refresh() + time.sleep(3) + activate = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", activate) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', activate) + print('Clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + self.driver.quit() + return False \ No newline at end of file diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py new file mode 100644 index 00000000..25f737b5 --- /dev/null +++ b/app/api/utils/yellowlab.py @@ -0,0 +1,133 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Yellowlab(): + + """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + + + def __init__(self, site=None): + self.site = site + + + def init_audit(self): + proc = subprocess.Popen([ + 'yellowlabtools', + self.site.site_url, + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + audits[cat].append(audit) + except: + pass + + + # get scores from each category + globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + scores = { + "globalScore": globalScore, + "pageWeight": pageWeight_score, + "requests": requests_score, + "domComplexity": domComplexity_score, + "javascriptComplexity": javascriptComplexity_score, + "badJavascript": badJavascript_score, + "jQuery": jQuery_score, + "cssComplexity": cssComplexity_score, + "badCSS": badCSS_score, + "fonts": fonts_score, + "serverConfig": serverConfig_score, + } + + data = { + "scores": scores, + "audits": audits + } + + except Exception as e: + print(e) + + scores = { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + data = { + "scores": scores, + "audits": audits + } + + return data + + diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index fd8559dc..fe21338d 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -1,4 +1,4 @@ -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from ...models import * from rest_framework import serializers from rest_framework.fields import UUIDField @@ -39,7 +39,8 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'scores', 'audits' + 'html', 'logs', 'lighthouse', 'yellowlab', 'images', + 'configs', ] @@ -50,7 +51,8 @@ class SmallScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 'scores' + fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', + 'lighthouse', 'yellowlab', 'configs', ] @@ -64,7 +66,7 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'scores_delta' + 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', ] @@ -77,7 +79,8 @@ class SmallTestSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', - 'pre_scan', 'post_scan', 'score', 'scores_delta' + 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', + 'yellowlab_delta', ] @@ -91,7 +94,7 @@ class Meta: model = Schedule fields = ['id', 'site', 'time_created', 'user', 'task_type', 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', - 'periodic_task_id', 'status', 'automation' + 'periodic_task_id', 'status', 'automation', 'extras' ] @@ -105,4 +108,18 @@ class Meta: model = Automation fields = ['id', 'expressions', 'actions', 'user', 'schedule', 'time_created', 'name' + ] + + + + +class ReportSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Report + fields = ['id', 'site', 'user', 'time_created', 'type', + 'path', 'info' ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 347a39ad..fe24f121 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,21 +1,19 @@ -import json, datetime +import json, boto3 +from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask -from ...models import (Test, Site, Scan, Log, Automation) +from django.shortcuts import get_object_or_404 +from ...models import * from rest_framework.response import Response from rest_framework import status -from ...models import (Test, Site, Scan, Log, Schedule, Account) -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer, SmallTestSerializer, - SmallScanSerializer, - ) +from .serializers import * +from ...tasks import * from rest_framework.pagination import LimitOffsetPagination -from django.urls import resolve -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.lighthouse import Lighthouse -from ...scan_tests.tester import Test as T -from ...tasks import (create_site_bg, create_scan_bg, create_test_bg) +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.image import Image as I +from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W @@ -94,7 +92,7 @@ def create_site(request, delay=False): if delay == True: create_site_bg.delay(site.id) else: - ScanSite(site=site).first_scan() + S(site=site).first_scan() serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -105,12 +103,16 @@ def create_site(request, delay=False): + + + + def get_sites(request): site_id = request.query_params.get('site_id') user = request.user if site_id != None: - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) if site.user != user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -133,13 +135,17 @@ def get_sites(request): def delete_site(request, id): user = request.user - site = Site.objects.get(id=id) + site = get_object_or_404(Site, pk=id) if site.user != user: data = {'reason': 'you cannot delete Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + # remove s3 objects + delete_site_s3_bg.delay(site_id=id) + + # remove site site.delete() data = {'message': 'Site has been deleted',} @@ -162,27 +168,82 @@ def create_test(request, delay=False): user = request.user site = Site.objects.get(id=site_id, ) if site.user != user: - data = {'reason': 'you cannot create a Test of a Site you do not own',} + data = {'reason': 'you cannot create a Test of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + + # get data from request + configs = request.data.get('configs', None) + pre_scan_id = request.data.get('pre_scan', None) + post_scan_id = request.data.get('post_scan', None) + index = request.data.get('index', None) + test_type = request.data.get('type', ['full']) + pre_scan = None + post_scan = None + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + + if pre_scan_id: + pre_scan = Scan.objects.get(id=pre_scan_id) + if post_scan_id: + post_scan = Scan.objects.get(id=post_scan_id) + + + # creating test object + test = Test.objects.create( + site=site, + type=test_type, + ) + if delay == True: - create_test_bg.delay(site.id) - data = {'message': 'test is being created in the background'} + create_test_bg.delay( + test_id=test.id, + configs=configs, + type=test_type, + index=index, + pre_scan=pre_scan_id, + post_scan=post_scan_id + ) + data = { + 'message': 'test is being created in the background', + 'id': str(test.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) + else: - test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() + post_scan.save() + + # updating test object + test.type = test_type + test.type = test_type test.pre_scan = pre_scan test.post_scan = post_scan test.save() - updated_test = T(test=test).run_full_test() + # running tester + updated_test = T(test=test).run_test(index=index) serializer_context = {'request': request,} serialized = TestSerializer(updated_test, context=serializer_context) @@ -194,6 +255,10 @@ def create_test(request, delay=False): + + + + def get_tests(request): user = request.user test_id = request.query_params.get('test_id') @@ -203,7 +268,7 @@ def get_tests(request): small = request.query_params.get('small') if test_id != None: - test = Test.objects.get(id=test_id) + test = get_object_or_404(Test, pk=test_id) if test.site.user != user: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} @@ -265,7 +330,7 @@ def get_tests(request): def delete_test(request, id): - test = Test.objects.get(id=id) + test = get_object_or_404(Test, pk=id) site = test.site user = request.user @@ -289,7 +354,7 @@ def create_scan(request, delay=False): site_id = request.data['site_id'] user = request.user - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) account_is_active = check_account(request) if not account_is_active: @@ -302,15 +367,30 @@ def create_scan(request, delay=False): record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + configs = request.data.get('configs', None) + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + # creating scan obj + created_scan = Scan.objects.create(site=site) + if delay == True: - create_scan_bg.delay(site.id) - data = {'message': 'scan is being created in the background'} + create_scan_bg.delay(scan_id=created_scan.id, configs=configs) + data = { + 'message': 'scan is being created in the background', + 'id': str(created_scan.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) else: - created_scan = Scan.objects.create(site=site) - updated_scan = ScanSite(scan=created_scan).first_scan() - + updated_scan = S(scan=created_scan, configs=configs).first_scan() serializer_context = {'request': request,} serialized = ScanSerializer(updated_scan, context=serializer_context) data = serialized.data @@ -332,7 +412,7 @@ def get_scans(request): small = request.query_params.get('small') if scan_id != None: - scan = Scan.objects.get(id=scan_id) + scan = get_object_or_404(Scan, pk=scan_id) if scan.site.user != user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} @@ -346,7 +426,7 @@ def get_scans(request): return Response(data, status=status.HTTP_200_OK) try: - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) except: if site_id != None: data = {'reason': 'cannot find a site with that id',} @@ -389,7 +469,7 @@ def get_scans(request): def delete_scan(request, id): - scan = Scan.objects.get(id=id) + scan = get_object_or_404(Scan, pk=id) site = scan.site user = request.user @@ -432,24 +512,17 @@ def create_or_update_schedule(request): except: schedule = None - try: - schedule_status = request.data['status'] - except: - schedule_status = None - - try: - begin_date_raw = request.data['begin_date'] - time = request.data['time'] - timezone = request.data['timezone'] - freq = request.data['frequency'] - task_type = request.data['task_type'] - except: - pass - try: - schedule_id = request.data['schedule_id'] - except: - schedule_id = None + schedule_status = request.data.get('status', None) + begin_date_raw = request.data.get('begin_date', None) + time = request.data.get('time', None) + timezone = request.data.get('timezone', None) + freq = request.data.get('frequency', None) + task_type = request.data.get('task_type', None) + test_type = request.data.get('test_type', None) + configs = request.data.get('configs', None) + schedule_id = request.data.get('schedule_id', None) + if schedule_status != None and schedule != None: @@ -464,25 +537,46 @@ def create_or_update_schedule(request): schedule.save() # retriving object again to avoid cacheing issues schedule_new = Schedule.objects.get(id=request.data['schedule_id']) + + # if not status change, updating data else: + + if Automation.objects.filter(schedule=schedule).exists(): + automation = Automation.objects.filter(schedule=schedule)[0] + auto_id = automation.id + else: + auto_id = None + if task_type == 'test': task = 'api.tasks.create_test_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'type': test_type, + 'automation_id': str(auto_id) } if task_type == 'scan': task = 'api.tasks.create_scan_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'automation_id': str(auto_id) + } + + if task_type == 'report': + task = 'api.tasks.create_report_bg' + arguments = { + 'site_id': str(site.id), + 'automation_id': str(auto_id) } format_str = '%m/%d/%Y' try: - begin_date = datetime.datetime.strptime(begin_date_raw, format_str) + begin_date = datetime.strptime(begin_date_raw, format_str) except: - begin_date = datetime.datetime.now() + begin_date = datetime.now() num_day_of_week = begin_date.weekday() day = begin_date.strftime("%d") @@ -513,12 +607,9 @@ def create_or_update_schedule(request): periodic_task.update( crontab=crontab, name=task_name, task=task, + kwargs=json.dumps(arguments), ) periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - elif PeriodicTask.objects.filter(name=task_name).exists(): - data = {'reason': 'Task has already be created',} - record_api_call(request, data, '401') - return Response(data, status=status.HTTP_401_UNAUTHORIZED) else: periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, @@ -526,11 +617,20 @@ def create_or_update_schedule(request): ) else: + if PeriodicTask.objects.filter(name=task_name).exists(): + data = {'reason': 'Task has already be created',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_401_UNAUTHORIZED) + periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, kwargs=json.dumps(arguments), ) + extras = { + "configs": configs, + "test_type": test_type, + } if schedule: schedule_query = Schedule.objects.filter(id=schedule_id) @@ -539,6 +639,7 @@ def create_or_update_schedule(request): user=request.user, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, task_type=task_type, + extras=extras ) schedule_new = Schedule.objects.get(id=schedule_id) else: @@ -546,7 +647,8 @@ def create_or_update_schedule(request): user=request.user, site=site, task_type=task_type, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, - periodic_task_id=periodic_task.id, + periodic_task_id=periodic_task.id, + extras=extras ) serializer_context = {'request': request,} @@ -565,7 +667,7 @@ def get_schedules(request): if schedule_id != None: - schedule = Schedule.objects.get(id=schedule_id) + schedule = get_object_or_404(Schedule, pk=schedule_id) if schedule.site.user != user or schedule.user != user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} @@ -580,7 +682,7 @@ def get_schedules(request): try: - site = Site.objects.get(id=site_id) + site = get_object_or_404(Site, pk=site_id) except: if site_id != None: data = {'reason': 'cannot find a site with that id',} @@ -612,7 +714,7 @@ def get_schedules(request): def delete_schedule(request, id): - schedule = Schedule.objects.get(id=id) + schedule = get_object_or_404(Schedule, pk=id) task = PeriodicTask.objects.get(id=schedule.periodic_task_id) site = schedule.site user = request.user @@ -688,6 +790,8 @@ def create_or_update_automation(request): arguments = { 'site_id': str(schedule.site.id), 'automation_id': str(automation.id), + 'configs': json.loads(task.kwargs).get('configs', None), + 'type': json.loads(task.kwargs).get('type', None), } task.kwargs=json.dumps(arguments) task.save() @@ -704,7 +808,7 @@ def get_automations(request): automation_id = request.query_params.get('automation_id') user = request.user if automation_id != None: - automation = Automation.objects.get(id=automation_id) + automation = get_object_or_404(Automation, pk=automation_id) if automation.user != user: data = {'reason': 'you cannot retrieve an Automation you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -726,7 +830,7 @@ def get_automations(request): def delete_automation(request, id): - automation = Automation.objects.get(id=id) + automation = get_object_or_404(Automation, pk=automation_id) if automation.user != request.user: data = {'reason': 'you cannot delete an automation you do not own',} @@ -743,6 +847,105 @@ def delete_automation(request, id): + + + + + +def create_or_update_report(request): + + report_id = request.data.get('report_id', None) + site_id = request.data.get('site_id', None) + report_type = request.data.get('type', ['full']) + text_color = request.data.get('text_color', '#24262d') + background_color = request.data.get('background_color', '#e1effd') + highlight_color = request.data.get('highlight_color', '#4283f8') + site = Site.objects.get(id=site_id) + + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + } + + if report_id: + report = get_object_or_404(Report, pk=report_id) + else: + report = Report.objects.create( + user=request.user, site=site + ) + + # update report data + report.info = info + report.type = report_type + report.save() + un_cached_report = Report.objects.get(id=report.id) + + + # generate report + updated_report = R(report=un_cached_report).make_test_report() + + + serializer_context = {'request': request,} + data = ReportSerializer(updated_report, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + +def get_reports(request): + site_id = request.query_params.get('site_id', None) + report_id = request.query_params.get('report_id', None) + + if site_id: + site = get_object_or_404(Site, pk=site_id) + reports = Report.objects.filter(site=site, user=request.user).order_by('-time_created') + + if report_id: + reports = get_object_or_404(Report, pk=report_id) + + if site_id is None and report_id is None: + reports = Report.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(reports, request) + serializer_context = {'request': request,} + serialized = ReportSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + + +def delete_report(request, id): + user = request.user + report = get_object_or_404(Report, pk=id) + + if report.user != user: + data = {'reason': 'you cannot delete Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_report_s3_bg.delay(report_id=id) + + # remove report + report.delete() + + data = {'message': 'Report has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + def get_logs(request): log_id = request.query_params.get('log_id') @@ -781,6 +984,76 @@ def get_logs(request): + +def install_wp_plugin(request): + login_url = request.data.get('login_url', None) + admin_url = request.data.get('admin_url', None) + plugin_name = request.data.get('plugin_name', None) + username = request.data.get('username', None) + password = request.data.get('password', None) + wait_time = request.data.get('wait_time', 15) + + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ) + + # login + wp_status = wp.login() + + # adjust lang + wp_status = wp.begin_lang_check() + + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + + # re adjust lang + wp_status = wp.end_lang_check() + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + + + + +def create_site_screenshot(request): + user = request.user + site_id = request.data.get('site_id', None) + url = request.data.get('url', None) + configs = request.data.get('configs', None) + site = None + + if site_id is not None: + site = Site.objects.get(id=site_id) + + data = I().screenshot(site=site, url=url, configs=configs) + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + def get_home_stats(request): sites = Site.objects.filter(user=request.user) site_count = sites.count() diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 5315a86a..ee707830 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,38 +1,150 @@ -from ...models import (Test, Site, Scan, Log) -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.tester import Test as T -from ...scan_tests.automations import automation +from ...models import * +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.reporter import Reporter as R +from ...utils.automations import automation +import boto3 +from scanerr import settings + def create_site_task(site_id): site = Site.objects.get(id=site_id) - ScanSite(site=site).first_scan() + S(site=site).first_scan() return site -def create_scan_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - created_scan = Scan.objects.create(site=site) - scan = ScanSite(scan=created_scan).first_scan() +def create_scan_task( + scan_id=None, + site_id=None, + automation_id=None, + type=None, + configs=None, + ): + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_scan = Scan.objects.create( + site=site, + ) + scan = S(scan=created_scan, configs=configs).first_scan() if automation_id: automation(automation_id, scan.id) return scan -def create_test_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - created_test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + + +def create_test_task( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + ): + + if test_id is not None: + created_test = Test.objects.get(id=test_id) + site = created_test.site + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_test = Test.objects.create( + site=site, + type=type, + ) + + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + pre_scan = Scan.objects.get(id=pre_scan) + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() + post_scan.save() + + # updating test object + created_test.type = type created_test.pre_scan = pre_scan created_test.post_scan = post_scan created_test.save() - test = T(test=created_test).run_full_test() + + + test = T(test=created_test).run_test(index=index) if automation_id: automation(automation_id, test.id) return test + + +def create_report_task(site_id, automation_id=None): + site = Site.objects.get(id=site_id) + if Report.objects.filter(site=site).exists(): + report = Report.objects.filter(site=site).order_by('-time_created')[0] + else: + info = { + "text_color": '#24262d', + "background_color": '#e1effd', + "highlight_color": '#4283f8', + } + report = Report.objects.create( + user=site.user, + site=site, + info=info, + ) + + + report = R(report=report).make_test_report() + if automation_id: + automation(automation_id, report.id) + return report + + + + + + +def delete_site_s3(site_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + + return + + + +def delete_report_s3(report_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # get site + site = Report.objects.get(id=report_id).site + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() + + return diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index 243f95b2..bc2cc034 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -18,5 +18,10 @@ path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), path('automation', views.Automations.as_view(), name='automation'), path('automation/', views.AutomationDetail.as_view(), name='automation-detail'), + path('report', views.Reports.as_view(), name='report'), + path('report/', views.ReportDetail.as_view(), name='report-detail'), path('home-stats', views.HomeStats.as_view(), name='home-stats'), + path('beta/wordpress/install-plugin', views.WordPressPluginInstall.as_view(), name='install-plugin'), + path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), + ] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 2fd4088f..b77cd629 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -2,7 +2,8 @@ from rest_framework.response import Response from rest_framework import status from django.contrib.auth.models import User -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from django.shortcuts import get_object_or_404 +from ...models import * from django.urls import path, include from rest_framework import routers, serializers, viewsets from rest_framework.viewsets import ViewSet @@ -10,18 +11,10 @@ from rest_framework.views import APIView from rest_framework.permissions import IsAuthenticated from django.views.decorators.csrf import ensure_csrf_cookie -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer - ) from rest_framework.pagination import LimitOffsetPagination -from .services import ( - record_api_call, create_or_update_schedule, create_test, get_tests, delete_test, - create_scan, get_scans, delete_scan, get_logs, create_site, get_sites, delete_site, - create_or_update_automation, get_automations, delete_automation, get_schedules, - delete_schedule, get_home_stats - ) from django.urls import resolve +from .serializers import * +from .services import * @@ -45,7 +38,7 @@ class SiteDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - site = Site.objects.get(id=id) + site = get_object_or_404(Site, pk=id) if site.user != request.user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -60,6 +53,7 @@ def delete(self, request, id): return response + class SiteDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -90,7 +84,7 @@ class ScanDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - scan = Scan.objects.get(id=id) + scan = get_object_or_404(Scan, pk=id) if scan.site.user != request.user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') @@ -140,7 +134,7 @@ class TestDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - test = Test.objects.get(id=id) + test = get_object_or_404(Test, pk=id) if test.site.user != request.user: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') @@ -189,7 +183,7 @@ class ScheduleDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - schedule = Schedule.objects.get(id=id) + schedule = get_object_or_404(Schedule, pk=id) if schedule.site.user != request.user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') @@ -228,7 +222,7 @@ class AutomationDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - automation = Automation.objects.get(id=id) + automation = get_object_or_404(Automation, pk=id) if automation.user != request.user: data = {'reason': 'you cannot retrieve Automations you do not own',} record_api_call(request, data, '403') @@ -248,6 +242,43 @@ def delete(self, request, id): +class Reports(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_report(request) + return response + + def get(self, request): + response = get_reports(request) + return response + + + +class ReportDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + report = get_object_or_404(Report, pk=id) + if report.user != request.user: + data = {'reason': 'you cannot retrieve Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ReportSerializer(report, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_report(request, id) + return response + + + class Logs(APIView): permission_classes = (AllowAny,) http_method_names = ['get',] @@ -263,7 +294,7 @@ class LogDetail(APIView): http_method_names = ['get',] def get(self, request, id): - log = Log.objects.get(id=id) + log = get_object_or_404(Log, pk=id) if log.user != request.user: data = {'reason': 'you cannot retrieve Logs you do not own',} record_api_call(request, data, '403') @@ -282,4 +313,24 @@ class HomeStats(APIView): def get(self, request): response = get_home_stats(request) + return response + + + + +class WordPressPluginInstall(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = install_wp_plugin(request) + return response + + +class SiteScreenshot(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_site_screenshot(request) return response \ No newline at end of file diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 6414f9ef..f15c2ff7 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -17,10 +17,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.environ.get('SECRET_KEY') @@ -49,6 +45,7 @@ 'rest_framework.authtoken', 'django_celery_beat', 'markdownify.apps.MarkdownifyConfig', + 'storages', ] MIDDLEWARE = [ @@ -88,12 +85,6 @@ if DEBUG == True: - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql @@ -107,12 +98,6 @@ else: - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql @@ -143,12 +128,12 @@ }, ] + # Django REST framework REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. 'DEFAULT_PERMISSION_CLASSES': [ - # 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', 'rest_framework.permissions.DjangoModelPermissions', ], @@ -160,12 +145,8 @@ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 10, - # 'DEFAULT_RENDERER_CLASSES': [ - # 'rest_framework.renderers.JSONRenderer', - # ], } - # SIMPLE_JWT = { # 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=.5), # 'REFRESH_TOKEN_LIFETIME': timedelta(minutes=1), @@ -193,6 +174,35 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") +# remote storage settings for serving static files to django admin +# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') +# STATIC_ROOT = 'static' +# MEDIA_ROOT = 'media' +# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" +# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" + +# Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + +# Configure which endpoint to send files to, and retrieve files from. +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') +AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') +AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') +AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') +AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') +AWS_LOCATION = os.environ.get('AWS_LOCATION') +AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') + +# General optimization for faster delivery +AWS_IS_GZIPPED = True +AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', +} + # Redis and Celery Conf @@ -217,5 +227,6 @@ EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') +# google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6f6f63a5..d0e3edb5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,6 +2,7 @@ version: '3' services: app: + privileged: true build: context: . dockerfile: Dockerfile.prod @@ -9,8 +10,8 @@ services: - ./app:/app - static_volume:/app/static command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && @@ -20,20 +21,12 @@ services: - 8000 env_file: - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data redis: image: redis:alpine celery: + privileged: true restart: always build: context: . @@ -43,7 +36,6 @@ services: env_file: - ./env/.env.prod depends_on: - # - db - redis - app @@ -78,7 +70,6 @@ services: volumes: - # pgdata: static_volume: certs: html: diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml deleted file mode 100644 index 2c8446cb..00000000 --- a/docker-compose.staging.yml +++ /dev/null @@ -1,94 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - # - db - - redis - - app - - - - nginx-proxy: - container_name: nginx-proxy - build: nginx - restart: always - ports: - - 443:443 - - 80:80 - volumes: - - static_volume:/app/static - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - - /var/run/docker.sock:/tmp/docker.sock:ro - depends_on: - - app - nginx-proxy-letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion - env_file: - - ./env/.env.staging.proxy-companion - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - depends_on: - - nginx-proxy - - - nginx: - build: ./nginx - ports: - - 80:8000 - # depends_on: - # - app - volumes: - - static_volume:/app/static - -volumes: - # pgdata: - static_volume: - certs: - html: - vhost: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 20f7df62..1ec86eba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: '3' services: app: + privileged: true build: context: . ports: @@ -9,8 +10,9 @@ services: volumes: - ./app:/app command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && python3 manage.py driver_test && @@ -29,6 +31,7 @@ services: redis: image: redis:alpine celery: + privileged: true restart: always build: context: . diff --git a/env/.env.dev.example b/env/.env.dev.example new file mode 100644 index 00000000..7327533f --- /dev/null +++ b/env/.env.dev.example @@ -0,0 +1,72 @@ +# high level django configs +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = http://localhost:3000 +API_URL_ROOT = http://localhost:8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_HOST = db +DB_NAME = app +DB_USER = postgres +DB_PASS = supersecretpassword +POSTGRES_DB = app +POSTGRES_USER = postgres +POSTGRES_PASSWORD = supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = +AWS_S3_REGION_NAME = +AWS_S3_ENDPOINT_URL = +AWS_S3_ENDPOINT_PATH = +AWS_S3_CUSTOM_DOMAIN = +AWS_S3_URL_PATH = +STORAGE_DOMAIN = +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example new file mode 100644 index 00000000..2ce982fa --- /dev/null +++ b/env/.env.prod.example @@ -0,0 +1,71 @@ +# high level django configs +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_NAME = defaultdb # example +DB_USER = doadmin # example +DB_PASS = +DB_PORT = +DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = +AWS_S3_REGION_NAME = +AWS_S3_ENDPOINT_URL = +AWS_S3_ENDPOINT_PATH = +AWS_S3_CUSTOM_DOMAIN = +AWS_S3_URL_PATH = +STORAGE_DOMAIN = +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.staging.proxy-companion b/env/.env.staging.proxy-companion deleted file mode 100644 index c51fdd4f..00000000 --- a/env/.env.staging.proxy-companion +++ /dev/null @@ -1,3 +0,0 @@ -DEFAULT_EMAIL=youremail@yourdomain.com -ACME_CA_URI=https://acme-staging-v02.api.letsencrypt.org/directory -NGINX_PROXY_CONTAINER=nginx-proxy \ No newline at end of file diff --git a/nginx_old/Dockerfile b/nginx_old/Dockerfile deleted file mode 100644 index 8e5916e0..00000000 --- a/nginx_old/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:1.19.0-alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d \ No newline at end of file diff --git a/nginx_old/custom.conf b/nginx_old/custom.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/custom.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/docker-compose.prod_old.yml b/nginx_old/docker-compose.prod_old.yml deleted file mode 100644 index fb41781e..00000000 --- a/nginx_old/docker-compose.prod_old.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - depends_on: - - db - - db: - image: postgres:10-alpine - env_file: - - ./env/.env.prod - volumes: - - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - - db - - redis - - app - - nginx: - build: ./nginx - ports: - - 80:8000 - depends_on: - - app - volumes: - - static_volume:/app/static - -volumes: - pgdata: - static_volume: \ No newline at end of file diff --git a/nginx_old/nginx.conf b/nginx_old/nginx.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/vhost.d/default b/nginx_old/vhost.d/default deleted file mode 100644 index c498447b..00000000 --- a/nginx_old/vhost.d/default +++ /dev/null @@ -1,9 +0,0 @@ - -location /static/ { - alias /app/static/; - add_header Access-Control-Allow-Origin *; -} - - - - diff --git a/requirements.txt b/requirements.txt index f240e00a..4df8b44c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0 +boto3==1.20.32 celery==5.1.0 certifi==2021.5.30 chardet==4.0.0 @@ -12,19 +13,22 @@ Django==3.2.3 django-celery-beat==2.2.0 django-filter==2.4.0 djangorestframework==3.12.4 +django-storages==1.12.3 docker==5.0.0 gunicorn==20.1.0 humanize==3.7.0 idna==2.10 kombu==5.1.0 Markdown==3.3.4 +Pillow==9.0.0 prometheus-client==0.8.0 prompt-toolkit==3.0.18 psycopg2==2.8.6 pytz==2021.1 redis==3.5.3 requests==2.25.1 -selenium==3.141.0 +reportlab==3.6.6 +selenium==4.1.0 six==1.16.0 sqlparse==0.4.1 tornado==6.1