diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml new file mode 100644 index 00000000..704b58cf --- /dev/null +++ b/.github/workflows/dev-deploy.yaml @@ -0,0 +1,34 @@ +name: Deploy to dev + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - dev + paths: + - 'app/**' + - 'Dockerfile' + - 'Dockerfile.dev' + - 'docker-compose.dev.yml' + - '.github/workflows/**' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: pulling latest dev commit and rebuilding app + uses: appleboy/ssh-action@v1.0.0 + with: + host: ${{ secrets.DEV_HOST }} + username: ${{ secrets.DEV_USER }} + key: ${{ secrets.DEV_KEY }} + command_timeout: 30m + script: | + cd ~/app + git pull origin dev + docker compose -f docker-compose.dev.yml down + docker volume rm app_app app_beat app_celery + docker compose -f docker-compose.dev.yml up -d --build \ No newline at end of file diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml new file mode 100644 index 00000000..aa16f7dc --- /dev/null +++ b/.github/workflows/k8s-deploy.yaml @@ -0,0 +1,76 @@ +# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and +# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, +# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions. + +name: Deploy to scanerr-k8s + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - main + paths: + - 'app/**' + - 'k8s/prod/**' + - 'Dockerfile' + - '.github/workflows/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel. +jobs: + # This workflow contains a single job called "build". + build: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + timeout-minutes: 30 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. + - name: Checkout main + uses: actions/checkout@main + + # Install doctl. + - name: Install doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + + # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA. + - name: Build container image + run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) . + + - name: Log in to Docker Hub Container Registry with short-lived credentialse + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + + - name: Push image to Docker Hub Container Registry + run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) + + # Update deployment files to reflect new build. + - name: Update app deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + + - name: Update celery deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml + + - name: Update beat deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml + + - name: Save DigitalOcean kubeconfig with short-lived credentials + run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} + + # Re-deploy app, celery and beat + - name: Deploy app + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + - name: Deploy celery + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml + - name: Deploy beat + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml + + - name: Verify app + run: kubectl rollout status deployment/app-deployment + - name: Verify celery + run: kubectl rollout status deployment/celery-deployment + - name: Verify beat + run: kubectl rollout status deployment/beat-deployment diff --git a/.gitignore b/.gitignore index 780009eb..e1253eb3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,29 @@ -app/api/utils/testing_stuff.py -app/data* -app/api/utils/__pycache__/tester.cpython-38.pyc +db.sqlite3 .DS_Store + *__pycache__* -db.sqlite3 *.pyc __pycache__ __pycache__/ */__pycache__/* **/__pycache__/ -server/app/env* + env/.env.local env/.env.dev env/.env.prod +env/.env.stage env/.env.prod.db + +app/data* app/static* -Dockerfile.alpine -Dockerfile.dev1 -Dockerfile.dev3 app/api/migrations/*_*.py + k8s/*/*-configs.yaml +k8s/prod/old_configs/* + +app/chromedriver* + +app/.cache* +app/.config* +app/.local* +app/.pki* diff --git a/Dockerfile b/Dockerfile index c1f55754..64a58d79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,115 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 +# pull main python image +FROM python:3.12-slim -# create the app user -RUN addgroup --system app && adduser --system app - -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip +# adding labels +LABEL Author="Scanerr" Support="hello@scanerr.io" -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig +# setting ENVs and Configs +ENV HOME=/app +ENV XDG_CACHE_HOME=$HOME/.cache +ENV DOCKERIZED=yes +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV MOZ_NO_REMOTE=1 +ENV MOZ_DISABLE_AUTO_SAFE_MODE=1 +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable +ENV PYTHONPATH="$HOME:$PYTHONPATH" +ENV NODE_OPTIONS="--max-old-space-size=4080" +ENV DJANGO_ALLOWED_HOSTS="*" +ENV SECRET_KEY="abcdefghijklmno123456789" -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ - && npm install -g n && n lts +# create the app user +RUN addgroup --system app && adduser --system app -# increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=4096" +# installing system deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql \ + postgresql-client \ + gcc \ + make \ + gfortran \ + openssl \ + libpq-dev \ + curl \ + libsm6 \ + libxrender1 \ + libxext6 \ + libgl1 \ + nasm \ + autoconf \ + libtool \ + automake \ + libjpeg-dev \ + libglib2.0-0 \ + libfreetype6 \ + ca-certificates \ + libfontconfig \ + apt-transport-https \ + software-properties-common + +# installing firefox-esr +RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +# installing google-chrome-stable +RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get install -y ./google-chrome-stable_current_amd64.deb && \ + rm google-chrome-stable_current_amd64.deb +# installing microsoft-edge-stable +RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \ + install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \ + sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \ + /etc/apt/sources.list.d/microsoft-edge.list' && \ + apt-get update && apt-get install -y microsoft-edge-stable -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# installing node and npm +RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \ + apt-get install -y --no-install-recommends nodejs && \ + npm install -g --no-cache n && \ + n lts -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes +# installing lighthouse & lighthouse-plugin-crux +RUN npm install -g lighthouse lighthouse-plugin-crux -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version +# installing lodash & yellowlabtools +RUN npm install -g lodash yellowlabtools -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt +# copying & installing requirements +COPY ./setup/requirements/requirements.txt /requirements.txt +RUN python3.12 -m pip install -r /requirements.txt # setting working dir -RUN mkdir /app COPY ./app /app WORKDIR /app +# setting browser cache dirs +RUN mkdir -p .mozilla .cache + # setting ownership RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file +RUN chown -R app:app /usr/local/bin/lighthouse +RUN chown -R app:app /usr/local/bin/yellowlabtools + +# writing migrations file +RUN python3.12 manage.py makemigrations --no-input + +# collecting static assets +RUN python3.12 manage.py collectstatic --no-input + +# cleaning up +RUN apt-get clean && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + microsoft.gpg + +# setting final user +USER app + +# staring up services +COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh" +ENTRYPOINT [ "/entrypoint.sh" ] + + + diff --git a/Dockerfile.prod b/Dockerfile.prod deleted file mode 100644 index eb8edf96..00000000 --- a/Dockerfile.prod +++ /dev/null @@ -1,52 +0,0 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 - -# create the app user -RUN addgroup --system app && adduser --system app - -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip - -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig git - -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y \ - && npm install -g n && n lts - -RUN npm cache clean --force - -# increasing allocated memory to node -# RUN export NODE_OPTIONS="--max-old-space-size=7000" -# RUN export NODE_OPTIONS="--stack-size=262000" -ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS="--max-old-space-size=7000" -# ENV NODE_OPTIONS=--stack-size=262000 - -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools - -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true - -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes - -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version - -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt - -# setting working dir -RUN mkdir /app -COPY ./app /app -WORKDIR /app - -# setting ownership -RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index ea474ece..97c65594 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,9 +1,11 @@ -Copyright (c) 2023 Scanerr +Copyright (c) 2024 Scanerr Scanerr Commercial Software License Terms 1. Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order. +(i) Software: Scanerr +(ii) Developer: Grey Labs, LLC (https://greylabs.io) 2. Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues. 3. Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software. 4. Billing. @@ -15,7 +17,7 @@ Scanerr Commercial Software License Terms 6. Use. (a) Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates. (b) Prohibited Uses. The customer may not: -(i) sell, lease, license, or sublicense the software or documentation +(i) sell, lease, license, or sublicense the software or documentation (ii) allow access to the software by others not licensed under this agreement (iii) share copies of the software or documentation with with others not licensed under this agreement (iv) make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers diff --git a/README.md b/README.md index d7c28503..08ac9f10 100644 --- a/README.md +++ b/README.md @@ -1,146 +1,18 @@ # Scanerr Server (API repo) -[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io) +![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yaml/badge.svg) -This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. +This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. -Copyright © Scanerr 2023 +Copyright © Scanerr 2024 --- -## Table of Contents +## Guides +- [Server Deployment](notes/Deployment.md) +- [k8s Deployment](notes/Kubernetes.md) - -#### Env's and deployment -- [Scanerr Server (API repo)](#scanerr-server-api-repo) - - [Table of Contents](#table-of-contents) - - [Env's and deployment](#envs-and-deployment) - - [Environment](#environment) - - [Local](#local) - - [Remote](#remote) - - [Scripts](#scripts) - - - - ---- - - -## Environment - -Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: -- high level django configs -- admin credentials -- email credentials -- database configs -- google API keys -- stripe keys -- OAuth keys -- twilio credentials -- slack credentials -- s3 remote storage credentials - - - ---- - - -## Local -Install and run locally on your machine in a dev environment. - -> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step. - -```shell -$ pip3 install virtualenv -$ virtualenv appenv -$ source appenv/bin/activate -$ mkdir app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose up --build -``` -*Spin-down the application* -```shell -$ docker compose up down -``` - - - ---- - - -## Remote -Install and deploy remotely in a production environment. - -> Ensure you have Docker installed and running on your server prior to this step. - -*Server configurations for Ubuntu 20.04* -``` shell -$ ssh root@your_server_ip -# apt update -# apt upgrade -# adduser {user} -# usermod -aG sudo {user} -# ufw allow OpenSSH -# ufw enable -# su {user} -``` - -*Add user to docker group* -```shell -$ sudo usermod -aG docker {user} -$ newgrp docker -``` - -*Generate SSH keys for GitHub* -``` shell -$ ssh-keygen -t ed25519 -C "your_github_email@example.com" -``` -- press `Enter` 3 times -```shell -$ eval "$(ssh-agent -s)" -$ ssh-add ~/.ssh/id_ed25519 -$ cat ~/.ssh/id_ed25519.pub -``` -- copy key to clipboard and paste in GutHub - - -*Create a dir to clone the app into* -``` shell -$ cd ~ -$ mkdir app -$ cd app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose -f docker-compose.prod.yml up -d --build -``` -*Spin-down the application* -```shell -$ docker compose -f docker-compose.prod.yml down -``` -*Spin-down the application and removes the volumes* -```shell -$ docker-compose -f docker-compose.prod.yml down -v -``` - - - - ---- - - - -## Scripts - -*ssh into container* -``` shell -$ docker exec -it /bin/sh -``` diff --git a/app/api/admin.py b/app/api/admin.py index 16e08a0b..8877f97f 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,34 +1,36 @@ from django.contrib import admin from .models import * from datetime import datetime +from .v1.ops.services import ( + create_scan, create_test, + delete_site, delete_page, + delete_scan, delete_test, + delete_case, delete_testcase, + crawl_site +) +from .tasks import ( + reset_account_usage, + update_scan_score +) -@admin.register(Site) -class SiteAdmin(admin.ModelAdmin): - list_display = ('site_url', 'user', 'time_created') - search_fields = ('site_url',) - - -@admin.register(Test) -class TestAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed', 'type') - search_fields = ('site',) -@admin.register(Scan) -class ScanAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed') - search_fields = ('site',) - actions = ['mark_as_completed',] - - def mark_as_completed(self, request, queryset): - queryset.update(time_completed=datetime.now()) @admin.register(Account) class AccountAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'type') search_fields = ('__str__',) + actions = ['reset_usage',] + + def reset_usage(self, request, queryset): + for account in queryset: + reset_account_usage.delay( + account_id=account.id + ) + + @admin.register(Member) @@ -37,25 +39,143 @@ class MemberAdmin(admin.ModelAdmin): search_fields = ('user__username', 'account__name') + + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) + + + +@admin.register(Site) +class SiteAdmin(admin.ModelAdmin): + list_display = ('site_url', 'account', 'time_created') + search_fields = ('site_url', 'account') + actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites'] + + def crawl_sites(self, request, queryset): + for site in queryset: + crawl_site( + id=site.id, + account=site.account + ) + + def scan_sites(self, request, queryset): + for site in queryset: + create_scan( + site_id=site.id, + user_id=site.account.user.id + ) + + def test_sites(self, request, queryset): + for site in queryset: + create_test( + site_id=site.id, + user_id=site.account.user.id + ) + + def delete_sites(self, request, queryset): + for site in queryset: + delete_site( + id=site.id, + account=site.account + ) + + + + +@admin.register(Page) +class SiteAdmin(admin.ModelAdmin): + list_display = ('page_url', 'account', 'time_created') + search_fields = ('page_url', 'account') + actions = ['scan_pages', 'test_pages', 'delete_pages',] + + def scan_pages(self, request, queryset): + for page in queryset: + create_scan( + page_id=page.id, + user_id=page.account.user.id + ) + + def test_pages(self, request, queryset): + for page in queryset: + create_test( + page_id=page.id, + user_id=page.account.user.id + ) + + def delete_pages(self, request, queryset): + for page in queryset: + delete_page( + id=page.id, + account=page.account + ) + + + + +@admin.register(Test) +class TestAdmin(admin.ModelAdmin): + list_display = ('id', 'page', 'time_created', 'time_completed', 'type') + search_fields = ('page',) + actions = ['delete_tests',] + + def delete_tests(self, request, queryset): + for test in queryset: + delete_test( + id=test.id, + account=test.page.account + ) + + + +@admin.register(Scan) +class ScanAdmin(admin.ModelAdmin): + list_display = ('id', 'page', 'time_created', 'time_completed') + search_fields = ('page',) + actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] # NEW!!! + + def delete_scans(self, request, queryset): + for scan in queryset: + delete_scan( + id=scan.id, + account=scan.page.account + ) + + def add_scan_score(self, request, queryset): # NEW!!! + for scan in queryset: + update_scan_score.delay( + scan_id=scan.id + ) + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + + + + @admin.register(Report) class ReportAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'user') + + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + + @admin.register(Schedule) class ScheduleAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'status', 'user') + list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created') + + @admin.register(Automation) @@ -63,19 +183,50 @@ class AutomationAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'schedule', 'user') + + @admin.register(Process) class ProcessAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') + list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'success') + + @admin.register(Case) class CaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created',) + actions = ['delete_cases',] + + def delete_cases(self, request, queryset): + for case in queryset: + delete_case( + id=case.id, + account=case.account + ) + + @admin.register(Testcase) class TestcaseAdmin(admin.ModelAdmin): list_display = ('__str__', 'user', 'time_created', 'time_completed',) + actions = ['delete_testcases',] + + def delete_testcases(self, request, queryset): + for testcase in queryset: + delete_testcase( + id=testcase.id, + account=testcase.account + ) + + + + +@admin.register(Issue) +class IssueAdmin(admin.ModelAdmin): + list_display = ('__str__', 'account', 'time_created', 'status',) + + @admin.register(Mask) @@ -88,4 +239,8 @@ def mark_as_inactive(self, request, queryset): queryset.update(active=False) def mark_as_active(self, request, queryset): - queryset.update(active=True) \ No newline at end of file + queryset.update(active=True) + + + + \ No newline at end of file diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 77c68bd6..e5f9a9c5 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -5,6 +5,12 @@ from ...utils.verify import verify import os + + + + + +# creates a new Admin user if None exists class Command(BaseCommand): def handle(self, *args, **options): diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py deleted file mode 100644 index c85d2705..00000000 --- a/app/api/management/commands/driver_p_test.py +++ /dev/null @@ -1,14 +0,0 @@ -from ...utils.driver_p import driver_test -from django.core.management.base import BaseCommand -import asyncio - -# testing puppeteer, pyppeteer, and chromium installation and configs - -class Command(BaseCommand): - - def handle(self, *args, **options): - asyncio.run(driver_test()) - - - - diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_test.py similarity index 83% rename from app/api/management/commands/driver_s_test.py rename to app/api/management/commands/driver_test.py index 3ae6438a..1fe02b82 100644 --- a/app/api/management/commands/driver_s_test.py +++ b/app/api/management/commands/driver_test.py @@ -1,8 +1,12 @@ -from ...utils.driver_s import driver_test +from ...utils.driver import driver_test from django.core.management.base import BaseCommand -# testing selenium, chromedriver, and chromium installation and configs + + + + +# testing selenium, chromedriver, and chromium installation and configs class Command(BaseCommand): def handle(self, *args, **options): diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py new file mode 100644 index 00000000..84804dfb --- /dev/null +++ b/app/api/management/commands/terminate_worker.py @@ -0,0 +1,74 @@ +from scanerr import celery +from django.core.management.base import BaseCommand +import time, os + + + + + + +# init warm shutdown (prevent new task acceptance) +class Command(BaseCommand): + + def handle(self, *args, **options): + + # get worker / pod name + default_worker = 'scanerr-celery' + if os.environ.get('THIS_POD_NAME'): + default_worker = str(os.environ.get('THIS_POD_NAME')) + + # get celery worker + this_worker = f"celery@{default_worker}" + + # sending initial SIGTERM to celery worker for warm-shutdown + celery.app.control.broadcast('shutdown', destination=[this_worker]) + + + + +# check if current tasks have completed +def wait_for_tasks_to_complete(): + + # get worker / pod name + default_worker = 'scanerr-celery' + if os.environ.get('THIS_POD_NAME'): + default_worker = str(os.environ.get('THIS_POD_NAME')) + + # get celery worker + this_worker = f"celery@{default_worker}" + + def get_task_list(): + + # set default + tasks = 0 + + try: + # Inspect all nodes. + i = celery.app.control.inspect() + + # Tasks received, but are still waiting to be executed. + reserved = i.reserved()[this_worker] + print(f'Reserved tasks -> {str(reserved)}') + + # Active tasks + active = i.active()[this_worker] + print(f'Active tasks -> {str(reserved)}') + + # Sum all tasks + tasks = len(active) + len(reserved) + + except Exception as e: + print(e) + + # return tasks count + return int(tasks) + + # get length of active and reserved task lists + tasks = get_task_list() + + # waiting for tasks to complete + while tasks > 0: + time.sleep(10) + tasks = get_task_list() + + \ No newline at end of file diff --git a/app/api/management/commands/wait_for_db.py b/app/api/management/commands/wait_for_db.py index c5edb897..def8f534 100644 --- a/app/api/management/commands/wait_for_db.py +++ b/app/api/management/commands/wait_for_db.py @@ -3,8 +3,13 @@ from django.db.utils import OperationalError from django.core.management import BaseCommand + + + + + +# Django command to pause execution until db is available class Command(BaseCommand): - """Django command to pause execution until db is available""" def handle(self, *args, **options): self.stdout.write('Waiting for database...') diff --git a/app/api/migrations/__init__.py b/app/api/migrations/__init__.py old mode 100644 new mode 100755 diff --git a/app/api/models.py b/app/api/models.py index 72e5b54d..a275773e 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -1,30 +1,35 @@ from django.db import models -from django.db import models from django.utils import timezone from django.urls import reverse from django.contrib.auth.models import User from datetime import datetime from django.contrib.postgres.fields import JSONField +from scanerr import settings import uuid + + + + def get_info_default(): info_default = { 'latest_scan': { 'id': None, 'time_created': None, 'time_completed': None, + 'score': None, }, 'latest_test': { 'id': None, 'time_created': None, 'time_completed': None, - 'score': None + 'score': None, + 'status': None }, 'lighthouse': { 'average': None, 'seo': None, - 'pwa': None, 'crux': None, 'performance': None, 'accessibility': None, @@ -33,7 +38,7 @@ def get_info_default(): 'yellowlab': { 'globalScore': None, 'pageWeight': None, - 'requests': None, + 'images': None, 'domComplexity': None, 'javascriptComplexity': None, 'badJavascript': None, @@ -42,17 +47,34 @@ def get_info_default(): 'badCSS': None, 'fonts': None, 'serverConfig': None, - }, - 'status': { - 'health': None, - 'badge': 'neutral', + } + } + return info_default + + + + +def get_small_info_default(): + info_default = { + 'latest_scan': { + 'id': None, + 'time_created': None, + 'time_completed': None, 'score': None, }, + 'latest_test': { + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None, + 'status': None + } } return info_default + def get_lh_delta_default(): lh_delta_default = { "scores": { @@ -60,22 +82,23 @@ def get_lh_delta_default(): "performance_delta": None, "accessibility_delta": None, "best-practices_delta": None, - "pwa_delta": None, "crux_delta": None, "average_delta" : None, "current_average": None, }, + "audits": None } return lh_delta_default + def get_yl_delta_default(): yl_delta_default = { "scores": { "average_delta": None, "pageWeight_delta": None, - "requests_delta": None, + "images_delta": None, "domComplexity_delta": None, "javascriptComplexity_delta": None, "badJavascript_delta": None, @@ -85,11 +108,13 @@ def get_yl_delta_default(): "fonts_delta": None, "serverConfig_delta": None, }, + "audits": None } return yl_delta_default + def get_lh_default(): lh_default = { "scores": { @@ -97,29 +122,22 @@ def get_lh_default(): "performance": None, "accessibility": None, "best_practices": None, - "pwa": None, "crux": None, "average": None }, - "audits": { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [], - "pwa": [], - "crux": [] - }, + "audits": None, } return lh_default + def get_yl_default(): yl_default = { "scores": { "globalScore": None, "pageWeight": None, - "requests": None, + "images": None, "domComplexity": None, "javascriptComplexity": None, "badJavascript": None, @@ -129,23 +147,13 @@ def get_yl_default(): "fonts": None, "serverConfig": None, }, - "audits": { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - }, + "audits": None, } return yl_default + def get_expressions_default(): expressions_default = { 'list': [ @@ -161,6 +169,7 @@ def get_expressions_default(): + def get_actions_default(): actions_default = { 'list': [ @@ -199,6 +208,7 @@ def get_steps_default(): + def get_scores_default(): scores_default = { 'html': None, @@ -211,6 +221,7 @@ def get_scores_default(): + def get_slack_default(): slack_default = { "slack_name": None, @@ -223,6 +234,8 @@ def get_slack_default(): return slack_default + + def get_tags_default(): tags_default = None, return tags_default @@ -230,20 +243,50 @@ def get_tags_default(): +def get_default_configs(): + configs = settings.CONFIGS + return configs + + + + +def get_usage_default(): + usage = { + 'scans': 0, + 'tests': 0, + 'testcases': 0, + 'scans_allowed': 30, + 'tests_allowed': 30, + 'testcases_allowed': 15, + } + return usage + + + + class Account(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) active = models.BooleanField(default=False, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free') code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) + max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) + max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0) + retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3) cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0) + interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month') + usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default) slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) + configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs) + meta = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): return self.user.email @@ -282,14 +325,15 @@ def __str__(self): - class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True) + time_crawl_completed = models.DateTimeField(serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): @@ -297,16 +341,35 @@ def __str__(self): + +class Page(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.page_url}' + + + + class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - html = models.TextField(serialize=True, null=True, blank=True) + html = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) + score = models.FloatField(serialize=True, null=True, blank=True) lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) configs = models.JSONField(serialize=True, null=True, blank=True) @@ -317,17 +380,21 @@ def __str__(self): + class Test(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) + threshold = models.FloatField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, null=True, blank=True) # working, failed, passed component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default) - html_delta = models.JSONField(serialize=True, null=True, blank=True) + html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) @@ -337,48 +404,45 @@ class Test(models.Model): post_scan_configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__test' + return f'{self.id}_test' -class Schedule(models.Model): +class Case(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') - time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase - timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) - begin_date = models.DateTimeField(default=datetime.now, serialize=True) - time = models.CharField(max_length=100, null=True, blank=True, serialize=True) - frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly, - task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task - crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) - periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) - status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) - extras = models.JSONField(serialize=True, null=True, blank=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + processed = models.BooleanField(default=False, serialize=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.site.site_url}__{self.task_type}' + return f'{self.name}' if len(self.name) > 0 else str(id) -class Automation(models.Model): +class Testcase(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + case_name = models.CharField(max_length=500, null=True, blank=True, serialize=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') - expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) - actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) + time_completed = models.DateTimeField(null=True, blank=True, serialize=True) + status = models.CharField(max_length=20, default='working', null=True, blank=True, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True) + configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.name}' - + return f'{self.case.name}_testcase' @@ -386,6 +450,7 @@ def __str__(self): class Report(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) @@ -394,45 +459,68 @@ class Report(models.Model): info = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__report' + return f'{self.page.page_url}_report' + +class Issue(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + trigger = models.JSONField(serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + details = models.TextField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, default='open') + affected = models.JSONField(serialize=True, null=True, blank=True) + labels = models.JSONField(serialize=True, null=True, blank=True) + read = models.BooleanField(default=False, serialize=True) + + def __str__(self): + return f'{self.title if self.title is not None else self.id}_issue' -class Case(models.Model): + + +class Schedule(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - time_created = models.DateTimeField(default=timezone.now, serialize=True) - steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) - tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + scope = models.CharField(max_length=100, default='account', serialize=True) + resources = models.JSONField(serialize=True, null=True, blank=True) + automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') + time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) + time_last_run = models.DateTimeField(null=True, blank=True, serialize=True) + task_type = models.CharField(max_length=100, default='test', serialize=True) + timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) + begin_date = models.DateTimeField(default=datetime.now, serialize=True) + time = models.CharField(max_length=100, null=True, blank=True, serialize=True) + frequency = models.CharField(default="monthly", max_length=100, serialize=True) + task = models.CharField(max_length=500, null=True, blank=True, serialize=True) + crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.name}' + return f'{self.account.name}_{self.task_type}_{self.scope}' -class Testcase(models.Model): +class Automation(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True) - site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - time_completed = models.DateTimeField(null=True, blank=True, serialize=True) - passed = models.BooleanField(default=False, serialize=True) - steps = models.JSONField(serialize=True, null=True, blank=True) - configs = models.JSONField(serialize=True, null=True, blank=True) + schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') + expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) + actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) def __str__(self): - return f'{self.case.name}__testcase' - + return f'{self.id}' - @@ -443,7 +531,7 @@ class Mask(models.Model): mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__mask' + return f'{self.id}_mask' @@ -451,15 +539,19 @@ def __str__(self): class Process(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + object_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - successful = models.BooleanField(serialize=True, default=False) + success = models.BooleanField(serialize=True, default=False) + exception = models.TextField(serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True) info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) progress = models.FloatField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__process' + return f'{self.id}_process' @@ -475,4 +567,7 @@ class Log(models.Model): response_payload = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.status}__{self.request_type}__{self.path}' + return f'{self.status}_{self.request_type}_{self.path}' + + + diff --git a/app/api/tasks.py b/app/api/tasks.py index d242b559..a9efd46d 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,143 +1,1627 @@ -from __future__ import absolute_import, unicode_literals -from typing import Any from celery.utils.log import get_task_logger -from celery import shared_task -from celery import Task as BaseTask -from .v1.ops.tasks import ( - create_site_task, create_scan_task, run_html_and_logs_task, - run_vrt_task, run_lighthouse_task, run_yellowlab_task, - create_test_task, create_report_task, delete_report_s3, - delete_site_s3, create_testcase_task, migrate_site_task, - delete_testcase_s3, - +from celery import shared_task, Task +from .utils.crawler import Crawler +from .utils.scanner import Scanner as S +from .utils.tester import Tester as T +from .utils.reporter import Reporter as R +from .utils.wordpress import Wordpress as W +from .utils.automater import Automater +from .utils.caser import Caser +from .utils.autocaser import AutoCaser +from .utils.exporter import create_and_send_report_export +from .utils.scanner import ( + _html_and_logs, _vrt, _lighthouse, + _yellowlab ) -from .models import Log +from .utils.alerts import send_invite_link, send_remove_alert +from .models import * from django.contrib.auth.models import User -from .utils.driver_p import driver_test -from asgiref.sync import async_to_sync -import asyncio +from django.utils import timezone +from datetime import datetime, timedelta +from scanerr import settings +import asyncio, boto3, time, requests, json, stripe -logger = get_task_logger(__name__) -@shared_task -def test_pupeteer(): - asyncio.run(driver_test()) - logger.info('Tested pupeteer instalation') -@shared_task -def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs): - create_site_task(site_id, scan_id, configs) - logger.info('Created scan of new site') +class BaseTaskWithRetry(Task): + autoretry_for = (Exception, KeyError) + retry_kwargs = {'max_retries': 2} + retry_backoff = True -@shared_task -def create_scan_bg( - scan_id=None, - site_id=None, - type=['full'], - automation_id=None, - configs=None, - tags=None, - *args, - **kwargs, - ): - create_scan_task( - scan_id, - site_id, - type, - automation_id, - configs, - tags, + + + +# setting logger +logger = get_task_logger(__name__) + + + + +# setting s3 instance +def s3(): + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) ) + return s3 + + + + +def check_and_increment_resource(account: object, resource: str) -> bool: + """ + Adds 1 to the Account.usage.{resource} if + {resource}_allowed has not been reached. + + Expcets: { + 'account' :

Your migration is complete!