diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml new file mode 100644 index 00000000..08cddef9 --- /dev/null +++ b/.github/workflows/dev-deploy.yaml @@ -0,0 +1,33 @@ +name: Deploy to dev + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - dev + paths: + - 'app/**' + - 'Dockerfile' + - 'docker-compose.dev.yml' + - '.github/workflows/**' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: pulling latest dev commit and rebuilding app + uses: appleboy/ssh-action@v1.0.0 + with: + host: ${{ secrets.DEV_HOST }} + username: ${{ secrets.DEV_USER }} + key: ${{ secrets.DEV_KEY }} + command_timeout: 30m + script: | # docker volume rm app_pgdata + cd ~/app + git pull origin dev + docker compose -f docker-compose.dev.yml down + docker volume rm app_server app_beat app_celery + docker compose -f docker-compose.dev.yml up -d --build \ No newline at end of file diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml new file mode 100644 index 00000000..9b8e5aff --- /dev/null +++ b/.github/workflows/k8s-deploy.yaml @@ -0,0 +1,79 @@ +# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and +# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, +# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions. + +name: Deploy to cursion-k8s + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - main + paths: + - 'app/**' + - 'k8s/prod/**' + - 'Dockerfile' + - '.github/workflows/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel. +jobs: + # This workflow contains a single job called "build". + build: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + timeout-minutes: 30 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. + - name: Checkout main + uses: actions/checkout@main + + # Install doctl. + - name: Install doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + + # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA. + - name: Build container image + run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) -t ${{ secrets.REGISTRY_NAME }}/server:latest . + + - name: Log in to Docker Hub Container Registry with short-lived credentialse + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + + - name: Push unique tag to Docker Hub Container Registry + run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) + + - name: Push latest tag to Docker Hub Container Registry + run: docker image push ${{ secrets.REGISTRY_NAME }}/server:latest + + # Update deployment files to reflect new build. + - name: Update app deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + + - name: Update celery deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml + + - name: Update beat deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml + + - name: Save DigitalOcean kubeconfig with short-lived credentials + run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} + + # Re-deploy app, celery and beat + - name: Deploy app + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + - name: Deploy celery + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml + - name: Deploy beat + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml + + - name: Verify app + run: kubectl rollout status deployment/app-deployment + - name: Verify celery + run: kubectl rollout status deployment/celery-deployment + - name: Verify beat + run: kubectl rollout status deployment/beat-deployment diff --git a/.gitignore b/.gitignore index 780009eb..2fddb707 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,34 @@ -app/api/utils/testing_stuff.py -app/data* -app/api/utils/__pycache__/tester.cpython-38.pyc +db.sqlite3 .DS_Store + *__pycache__* -db.sqlite3 *.pyc __pycache__ __pycache__/ */__pycache__/* **/__pycache__/ -server/app/env* + env/.env.local env/.env.dev env/.env.prod +env/.env.stage +env/.env.remote env/.env.prod.db +env/.env.client.dev +env/.env.client.prod + + +app/data* app/static* -Dockerfile.alpine -Dockerfile.dev1 -Dockerfile.dev3 app/api/migrations/*_*.py + k8s/*/*-configs.yaml +k8s/prod/old_configs/* + +app/chromedriver* + +app/.cache* +app/.config* +app/.local* +app/.pki* +env/.env.client.dev diff --git a/Dockerfile b/Dockerfile index c1f55754..204d1157 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,115 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 +# pull main python image +FROM python:3.12-slim -# create the app user -RUN addgroup --system app && adduser --system app - -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip +# adding labels +LABEL Author="Cursion" Support="hello@cursion.dev" -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig +# setting ENVs and Configs +ENV HOME=/app +ENV XDG_CACHE_HOME=$HOME/.cache +ENV DOCKERIZED=yes +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV MOZ_NO_REMOTE=1 +ENV MOZ_DISABLE_AUTO_SAFE_MODE=1 +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable +ENV PYTHONPATH="$HOME:$PYTHONPATH" +ENV NODE_OPTIONS="--max-old-space-size=4080" +ENV DJANGO_ALLOWED_HOSTS="*" +ENV SECRET_KEY="abcdefghijklmno123456789" -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ - && npm install -g n && n lts +# create the app user +RUN addgroup --system app && adduser --system app -# increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=4096" +# installing system deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql \ + postgresql-client \ + gcc \ + make \ + gfortran \ + openssl \ + libpq-dev \ + curl \ + libsm6 \ + libxrender1 \ + libxext6 \ + libgl1 \ + nasm \ + autoconf \ + libtool \ + automake \ + libjpeg-dev \ + libglib2.0-0 \ + libfreetype6 \ + ca-certificates \ + libfontconfig \ + apt-transport-https \ + software-properties-common + +# installing firefox-esr +RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +# installing google-chrome-stable +RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get install -y ./google-chrome-stable_current_amd64.deb && \ + rm google-chrome-stable_current_amd64.deb +# installing microsoft-edge-stable +RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \ + install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \ + sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \ + /etc/apt/sources.list.d/microsoft-edge.list' && \ + apt-get update && apt-get install -y microsoft-edge-stable -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# installing node and npm +RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \ + apt-get install -y --no-install-recommends nodejs && \ + npm install -g --no-cache n && \ + n lts -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes +# installing lighthouse & lighthouse-plugin-crux +RUN npm install -g lighthouse lighthouse-plugin-crux -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version +# installing lodash & yellowlabtools +RUN npm install -g lodash yellowlabtools -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt +# copying & installing requirements +COPY ./setup/requirements/requirements.txt /requirements.txt +RUN python3.12 -m pip install -r /requirements.txt # setting working dir -RUN mkdir /app COPY ./app /app WORKDIR /app +# setting browser cache dirs +RUN mkdir -p .mozilla .cache + # setting ownership RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file +RUN chown -R app:app /usr/local/bin/lighthouse +RUN chown -R app:app /usr/local/bin/yellowlabtools + +# writing migrations file +RUN python3.12 manage.py makemigrations --no-input + +# collecting static assets +RUN python3.12 manage.py collectstatic --no-input + +# cleaning up +RUN apt-get clean && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + microsoft.gpg + +# setting final user +USER app + +# staring up services +COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh" +ENTRYPOINT [ "/entrypoint.sh" ] + + + diff --git a/Dockerfile.prod b/Dockerfile.prod deleted file mode 100644 index eb8edf96..00000000 --- a/Dockerfile.prod +++ /dev/null @@ -1,52 +0,0 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 - -# create the app user -RUN addgroup --system app && adduser --system app - -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip - -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig git - -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y \ - && npm install -g n && n lts - -RUN npm cache clean --force - -# increasing allocated memory to node -# RUN export NODE_OPTIONS="--max-old-space-size=7000" -# RUN export NODE_OPTIONS="--stack-size=262000" -ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS="--max-old-space-size=7000" -# ENV NODE_OPTIONS=--stack-size=262000 - -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools - -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true - -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes - -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version - -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt - -# setting working dir -RUN mkdir /app -COPY ./app /app -WORKDIR /app - -# setting ownership -RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index ea474ece..4975f547 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,9 +1,11 @@ -Copyright (c) 2023 Scanerr +Copyright (c) 2024 Cursion -Scanerr Commercial Software License Terms +Cursion Commercial Software License Terms 1. Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order. +(i) Software: Cursion +(ii) Developer: Grey Labs, LLC (https://greylabs.io) 2. Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues. 3. Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software. 4. Billing. @@ -15,7 +17,7 @@ Scanerr Commercial Software License Terms 6. Use. (a) Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates. (b) Prohibited Uses. The customer may not: -(i) sell, lease, license, or sublicense the software or documentation +(i) sell, lease, license, or sublicense the software or documentation (ii) allow access to the software by others not licensed under this agreement (iii) share copies of the software or documentation with with others not licensed under this agreement (iv) make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers diff --git a/README.md b/README.md index d7c28503..ae820eff 100644 --- a/README.md +++ b/README.md @@ -1,146 +1,18 @@ -# Scanerr Server (API repo) +# Cursion Server (API repo) -[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io) +![Build Status](https://github.com/cursion-dev/server/actions/workflows/k8s-deploy.yaml/badge.svg) -This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. +This is the server repo for the Cursion API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. -Copyright © Scanerr 2023 +Copyright © Cursion 2024 --- -## Table of Contents +## Guides +- [Server Deployment](notes/Deployment.md) +- [k8s Deployment](notes/Kubernetes.md) - -#### Env's and deployment -- [Scanerr Server (API repo)](#scanerr-server-api-repo) - - [Table of Contents](#table-of-contents) - - [Env's and deployment](#envs-and-deployment) - - [Environment](#environment) - - [Local](#local) - - [Remote](#remote) - - [Scripts](#scripts) - - - - ---- - - -## Environment - -Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: -- high level django configs -- admin credentials -- email credentials -- database configs -- google API keys -- stripe keys -- OAuth keys -- twilio credentials -- slack credentials -- s3 remote storage credentials - - - ---- - - -## Local -Install and run locally on your machine in a dev environment. - -> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step. - -```shell -$ pip3 install virtualenv -$ virtualenv appenv -$ source appenv/bin/activate -$ mkdir app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose up --build -``` -*Spin-down the application* -```shell -$ docker compose up down -``` - - - ---- - - -## Remote -Install and deploy remotely in a production environment. - -> Ensure you have Docker installed and running on your server prior to this step. - -*Server configurations for Ubuntu 20.04* -``` shell -$ ssh root@your_server_ip -# apt update -# apt upgrade -# adduser {user} -# usermod -aG sudo {user} -# ufw allow OpenSSH -# ufw enable -# su {user} -``` - -*Add user to docker group* -```shell -$ sudo usermod -aG docker {user} -$ newgrp docker -``` - -*Generate SSH keys for GitHub* -``` shell -$ ssh-keygen -t ed25519 -C "your_github_email@example.com" -``` -- press `Enter` 3 times -```shell -$ eval "$(ssh-agent -s)" -$ ssh-add ~/.ssh/id_ed25519 -$ cat ~/.ssh/id_ed25519.pub -``` -- copy key to clipboard and paste in GutHub - - -*Create a dir to clone the app into* -``` shell -$ cd ~ -$ mkdir app -$ cd app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose -f docker-compose.prod.yml up -d --build -``` -*Spin-down the application* -```shell -$ docker compose -f docker-compose.prod.yml down -``` -*Spin-down the application and removes the volumes* -```shell -$ docker-compose -f docker-compose.prod.yml down -v -``` - - - - ---- - - - -## Scripts - -*ssh into container* -``` shell -$ docker exec -it /bin/sh -``` diff --git a/app/api/admin.py b/app/api/admin.py index 16e08a0b..a726d93d 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,81 +1,259 @@ from django.contrib import admin from .models import * from datetime import datetime +from .v1.ops.services import ( + create_scan, create_test, + delete_site, delete_page, + delete_scan, delete_test, + delete_case, delete_caserun, + crawl_site +) +from .tasks import ( + reset_account_usage, + update_scan_score +) -@admin.register(Site) -class SiteAdmin(admin.ModelAdmin): - list_display = ('site_url', 'user', 'time_created') - search_fields = ('site_url',) - - -@admin.register(Test) -class TestAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed', 'type') - search_fields = ('site',) -@admin.register(Scan) -class ScanAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed') - search_fields = ('site',) - actions = ['mark_as_completed',] - - def mark_as_completed(self, request, queryset): - queryset.update(time_completed=datetime.now()) @admin.register(Account) class AccountAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'type') search_fields = ('__str__',) + actions = ['reset_usage',] + + def reset_usage(self, request, queryset): + for account in queryset: + reset_account_usage.delay( + account_id=account.id + ) + + @admin.register(Member) class MemberAdmin(admin.ModelAdmin): - list_display = ('user', 'account', 'time_created', 'type', 'status') + list_display = ('email', 'account', 'time_created', 'type', 'status') search_fields = ('user__username', 'account__name') + + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) + + + +@admin.register(Site) +class SiteAdmin(admin.ModelAdmin): + list_display = ('site_url', 'account', 'time_created') + search_fields = ('site_url', 'account') + actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites'] + + def crawl_sites(self, request, queryset): + for site in queryset: + crawl_site( + id=site.id, + user=site.account.user + ) + + def scan_sites(self, request, queryset): + for site in queryset: + create_scan( + site_id=site.id, + user_id=site.account.user.id + ) + + def test_sites(self, request, queryset): + for site in queryset: + create_test( + site_id=site.id, + user_id=site.account.user.id + ) + + def delete_sites(self, request, queryset): + for site in queryset: + delete_site( + id=site.id, + user=site.account.user + ) + + + + +@admin.register(Page) +class SiteAdmin(admin.ModelAdmin): + list_display = ('page_url', 'account', 'time_created') + search_fields = ('page_url', 'account') + actions = ['scan_pages', 'test_pages', 'delete_pages',] + + def scan_pages(self, request, queryset): + for page in queryset: + create_scan( + page_id=page.id, + user_id=page.account.user.id + ) + + def test_pages(self, request, queryset): + for page in queryset: + create_test( + page_id=page.id, + user_id=page.account.user.id + ) + + def delete_pages(self, request, queryset): + for page in queryset: + delete_page( + id=page.id, + user=page.user + ) + + + + +@admin.register(Test) +class TestAdmin(admin.ModelAdmin): + list_display = ('id', 'page', 'time_created', 'time_completed', 'type') + search_fields = ('page__page_url',) + actions = ['delete_tests',] + + def delete_tests(self, request, queryset): + for test in queryset: + delete_test( + id=test.id, + account=test.page.account + ) + + + +@admin.register(Scan) +class ScanAdmin(admin.ModelAdmin): + list_display = ('id', 'page', 'time_created', 'time_completed') + search_fields = ('page__page_url',) + actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] + + def delete_scans(self, request, queryset): + for scan in queryset: + delete_scan( + id=scan.id, + account=scan.page.account + ) + + def add_scan_score(self, request, queryset): + for scan in queryset: + update_scan_score.delay( + scan_id=scan.id + ) + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + + + + @admin.register(Report) class ReportAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'user') + + @admin.register(Log) class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + + @admin.register(Schedule) class ScheduleAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'status', 'user') + list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created') + + -@admin.register(Automation) -class AutomationAdmin(admin.ModelAdmin): +@admin.register(Alert) +class AlertAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'schedule', 'user') + + @admin.register(Process) class ProcessAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') + list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'success') + + @admin.register(Case) class CaseAdmin(admin.ModelAdmin): - list_display = ('__str__', 'user', 'time_created',) + list_display = ('title', 'user', 'site', 'time_created',) + search_fields = ('title', 'site__site_url') + actions = ['delete_cases',] + + def delete_cases(self, request, queryset): + for case in queryset: + delete_case( + id=case.id, + user=case.user + ) + + + +@admin.register(CaseRun) +class CaseRunAdmin(admin.ModelAdmin): + list_display = ('title', 'user', 'time_created', 'time_completed',) + search_fields = ('title', 'site__site_url') + + actions = ['delete_caseruns',] + + def delete_caseruns(self, request, queryset): + for caserun in queryset: + delete_caserun( + id=caserun.id, + user=caserun.user + ) + + + + +@admin.register(Issue) +class IssueAdmin(admin.ModelAdmin): + list_display = ('title', 'account', 'time_created', 'status',) + search_fields = ('title', 'affected') + + + + +@admin.register(Flow) +class FlowAdmin(admin.ModelAdmin): + list_display = ('title', 'account', 'time_created',) + search_fields = ('title',) + + + + + +@admin.register(FlowRun) +class FlowRunAdmin(admin.ModelAdmin): + list_display = ('title', 'account', 'site', 'time_created', 'time_completed', 'status') + search_fields = ('title', 'site__site_url',) + + + + +@admin.register(Secret) +class SecretAdmin(admin.ModelAdmin): + list_display = ('__str__', 'account', 'time_created',) -@admin.register(Testcase) -class TestcaseAdmin(admin.ModelAdmin): - list_display = ('__str__', 'user', 'time_created', 'time_completed',) @admin.register(Mask) @@ -88,4 +266,8 @@ def mark_as_inactive(self, request, queryset): queryset.update(active=False) def mark_as_active(self, request, queryset): - queryset.update(active=True) \ No newline at end of file + queryset.update(active=True) + + + + \ No newline at end of file diff --git a/app/api/apps.py b/app/api/apps.py index 36985c8a..275aba4d 100644 --- a/app/api/apps.py +++ b/app/api/apps.py @@ -2,4 +2,7 @@ class ApiConfig(AppConfig): - name = 'api' \ No newline at end of file + name = 'api' + + def ready(self): + import api.signals \ No newline at end of file diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 77c68bd6..af9a1b3d 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -1,16 +1,23 @@ from django.core.management.base import BaseCommand from rest_framework.authtoken.models import Token from django.contrib.auth.models import User -from ...models import Account +from ...models import Account, Member, get_permissions_default from ...utils.verify import verify -import os +import os, secrets + + + + + +# creates a new Admin user if None exists class Command(BaseCommand): def handle(self, *args, **options): username = os.environ.get('ADMIN_USER') email = os.environ.get('ADMIN_EMAIL') password = os.environ.get('ADMIN_PASS') + mode = os.environ.get('MODE') if User.objects.filter(is_superuser=True).count() == 0: print('Creating Admin User for %s (%s)' % (username, email)) admin = User.objects.create_superuser(email=email, username=username, password=password) @@ -23,15 +30,52 @@ def handle(self, *args, **options): user = User.objects.get(username=username) if not Account.objects.filter(user=user).exists(): print('Funding account for %s' % (username)) - Account.objects.create( + + # default usage + usage = { + 'sites': 0, + 'schedules': 0, + 'scans': 0, + 'tests': 0, + 'caseruns': 0, + 'flowruns': 0, + 'sites_allowed': 1000, + 'pages_allowed': 10, + 'schedules_allowed': 50, + 'scans_allowed': 100000, + 'tests_allowed': 100000, + 'caseruns_allowed': 100000, + 'flowruns_allowed': 100000, + 'nodes_allowed': 50, + 'conditions_allowed': 25, + 'retention_days': 1000, + } + + code = secrets.token_urlsafe(16) + + account = Account.objects.create( + name='Admin', user=user, active=True, - type='enterprise', - max_sites=10000, + type='selfhost' if mode == 'selfhost' else 'admin', + usage=usage, + code=code, + ) + + # get permissonions or default + permissions = get_permissions_default() + + member = Member.objects.create( + user=user, + email=email, + status='active', + type='admin', + account=account, + permissions=permissions ) + else: print('Accounts can only be initialized if no Accounts exist') if not Token.objects.filter(user=user).exists(): - Token.objects.create(user=user) - # verify() \ No newline at end of file + Token.objects.create(user=user) \ No newline at end of file diff --git a/app/api/management/commands/create_tasks.py b/app/api/management/commands/create_tasks.py new file mode 100644 index 00000000..4eb52fd5 --- /dev/null +++ b/app/api/management/commands/create_tasks.py @@ -0,0 +1,58 @@ +from django.core.management.base import BaseCommand +from django_celery_beat.models import PeriodicTask, IntervalSchedule +from datetime import datetime + + + + + + +# creating default system tasks +class Command(BaseCommand): + + def handle(self, *args, **options): + + tasks = [ + { + 'every': 2, + 'period': IntervalSchedule.MINUTES, + 'name': 'Redeliver Failed Tasks', + 'task': 'api.tasks.redeliver_failed_tasks' + }, + { + 'every': 1, + 'period': IntervalSchedule.DAYS, + 'name': 'Data Retention Cleanup', + 'task': 'api.tasks.data_retention' + }, + { + 'every': 1, + 'period': IntervalSchedule.DAYS, + 'name': 'Reset Account Usage', + 'task': 'api.tasks.reset_account_usage' + }, + ] + + # loop through and create + # PeriodicTasks for each + for task in tasks: + + print(f'Setting up Task: {task.get('name')}') + + try: + # create the schedule + schedule, created = IntervalSchedule.objects.get_or_create( + every=task.get('every'), + period=task.get('period'), + ) + + # create the task + PeriodicTask.objects.create( + interval=schedule, + name=task.get('name'), + task=task.get('task') + ) + + except Exception as e: + print(e) + diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py deleted file mode 100644 index c85d2705..00000000 --- a/app/api/management/commands/driver_p_test.py +++ /dev/null @@ -1,14 +0,0 @@ -from ...utils.driver_p import driver_test -from django.core.management.base import BaseCommand -import asyncio - -# testing puppeteer, pyppeteer, and chromium installation and configs - -class Command(BaseCommand): - - def handle(self, *args, **options): - asyncio.run(driver_test()) - - - - diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py new file mode 100644 index 00000000..1cadfa8c --- /dev/null +++ b/app/api/management/commands/terminate_worker.py @@ -0,0 +1,74 @@ +from cursion import celery +from django.core.management.base import BaseCommand +import time, os + + + + + + +# init warm shutdown (prevent new task acceptance) +class Command(BaseCommand): + + def handle(self, *args, **options): + + # get worker / pod name + default_worker = 'cursion-celery' + if os.environ.get('THIS_POD_NAME'): + default_worker = str(os.environ.get('THIS_POD_NAME')) + + # get celery worker + this_worker = f"celery@{default_worker}" + + # sending initial SIGTERM to celery worker for warm-shutdown + celery.app.control.broadcast('shutdown', destination=[this_worker]) + + + + +# check if current tasks have completed +def wait_for_tasks_to_complete(): + + # get worker / pod name + default_worker = 'cursion-celery' + if os.environ.get('THIS_POD_NAME'): + default_worker = str(os.environ.get('THIS_POD_NAME')) + + # get celery worker + this_worker = f"celery@{default_worker}" + + def get_task_list(): + + # set default + tasks = 0 + + try: + # Inspect all nodes. + i = celery.app.control.inspect() + + # Tasks received, but are still waiting to be executed. + reserved = i.reserved()[this_worker] + print(f'Reserved tasks -> {str(reserved)}') + + # Active tasks + active = i.active()[this_worker] + print(f'Active tasks -> {str(reserved)}') + + # Sum all tasks + tasks = len(active) + len(reserved) + + except Exception as e: + print(e) + + # return tasks count + return int(tasks) + + # get length of active and reserved task lists + tasks = get_task_list() + + # waiting for tasks to complete + while tasks > 0: + time.sleep(10) + tasks = get_task_list() + + \ No newline at end of file diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/test_driver.py similarity index 83% rename from app/api/management/commands/driver_s_test.py rename to app/api/management/commands/test_driver.py index 3ae6438a..1fe02b82 100644 --- a/app/api/management/commands/driver_s_test.py +++ b/app/api/management/commands/test_driver.py @@ -1,8 +1,12 @@ -from ...utils.driver_s import driver_test +from ...utils.driver import driver_test from django.core.management.base import BaseCommand -# testing selenium, chromedriver, and chromium installation and configs + + + + +# testing selenium, chromedriver, and chromium installation and configs class Command(BaseCommand): def handle(self, *args, **options): diff --git a/app/api/management/commands/verify_account.py b/app/api/management/commands/verify_account.py new file mode 100644 index 00000000..a93de01b --- /dev/null +++ b/app/api/management/commands/verify_account.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand +from ...utils.verify import verify + + + + + + +# verifies deployment +class Command(BaseCommand): + + def handle(self, *args, **options): + verify() \ No newline at end of file diff --git a/app/api/management/commands/wait_for_db.py b/app/api/management/commands/wait_for_db.py index c5edb897..def8f534 100644 --- a/app/api/management/commands/wait_for_db.py +++ b/app/api/management/commands/wait_for_db.py @@ -3,8 +3,13 @@ from django.db.utils import OperationalError from django.core.management import BaseCommand + + + + + +# Django command to pause execution until db is available class Command(BaseCommand): - """Django command to pause execution until db is available""" def handle(self, *args, **options): self.stdout.write('Waiting for database...') diff --git a/app/api/migrations/__init__.py b/app/api/migrations/__init__.py old mode 100644 new mode 100755 diff --git a/app/api/models.py b/app/api/models.py index 72e5b54d..cd7474ef 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -1,11 +1,15 @@ from django.db import models -from django.db import models from django.utils import timezone from django.urls import reverse from django.contrib.auth.models import User -from datetime import datetime +from datetime import datetime, timezone as tz from django.contrib.postgres.fields import JSONField -import uuid +from cursion import settings +import uuid, secrets + + + + def get_info_default(): @@ -14,17 +18,18 @@ def get_info_default(): 'id': None, 'time_created': None, 'time_completed': None, + 'score': None, }, 'latest_test': { 'id': None, 'time_created': None, 'time_completed': None, - 'score': None + 'score': None, + 'status': None }, 'lighthouse': { 'average': None, 'seo': None, - 'pwa': None, 'crux': None, 'performance': None, 'accessibility': None, @@ -33,7 +38,7 @@ def get_info_default(): 'yellowlab': { 'globalScore': None, 'pageWeight': None, - 'requests': None, + 'images': None, 'domComplexity': None, 'javascriptComplexity': None, 'badJavascript': None, @@ -42,17 +47,34 @@ def get_info_default(): 'badCSS': None, 'fonts': None, 'serverConfig': None, - }, - 'status': { - 'health': None, - 'badge': 'neutral', + } + } + return info_default + + + + +def get_small_info_default(): + info_default = { + 'latest_scan': { + 'id': None, + 'time_created': None, + 'time_completed': None, 'score': None, }, + 'latest_test': { + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None, + 'status': None + } } return info_default + def get_lh_delta_default(): lh_delta_default = { "scores": { @@ -60,22 +82,23 @@ def get_lh_delta_default(): "performance_delta": None, "accessibility_delta": None, "best-practices_delta": None, - "pwa_delta": None, "crux_delta": None, "average_delta" : None, "current_average": None, }, + "audits": None } return lh_delta_default + def get_yl_delta_default(): yl_delta_default = { "scores": { "average_delta": None, "pageWeight_delta": None, - "requests_delta": None, + "images_delta": None, "domComplexity_delta": None, "javascriptComplexity_delta": None, "badJavascript_delta": None, @@ -85,11 +108,13 @@ def get_yl_delta_default(): "fonts_delta": None, "serverConfig_delta": None, }, + "audits": None } return yl_delta_default + def get_lh_default(): lh_default = { "scores": { @@ -97,29 +122,22 @@ def get_lh_default(): "performance": None, "accessibility": None, "best_practices": None, - "pwa": None, "crux": None, "average": None }, - "audits": { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [], - "pwa": [], - "crux": [] - }, + "audits": None, } return lh_default + def get_yl_default(): yl_default = { "scores": { "globalScore": None, "pageWeight": None, - "requests": None, + "images": None, "domComplexity": None, "javascriptComplexity": None, "badJavascript": None, @@ -129,23 +147,13 @@ def get_yl_default(): "fonts": None, "serverConfig": None, }, - "audits": { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - }, + "audits": None, } return yl_default + def get_expressions_default(): expressions_default = { 'list': [ @@ -161,6 +169,7 @@ def get_expressions_default(): + def get_actions_default(): actions_default = { 'list': [ @@ -199,6 +208,7 @@ def get_steps_default(): + def get_scores_default(): scores_default = { 'html': None, @@ -211,6 +221,7 @@ def get_scores_default(): + def get_slack_default(): slack_default = { "slack_name": None, @@ -223,6 +234,8 @@ def get_slack_default(): return slack_default + + def get_tags_default(): tags_default = None, return tags_default @@ -230,20 +243,154 @@ def get_tags_default(): +def get_default_configs(): + configs = settings.CONFIGS + return configs + + + + +def get_usage_default(): + usage = { + 'sites': 0, + 'schedules': 0, + 'scans': 0, + 'tests': 0, + 'caseruns': 0, + 'flowruns': 0, + 'sites_allowed': 1, + 'pages_allowed': 3, + 'schedules_allowed': 1, + 'scans_allowed': 30, + 'tests_allowed': 30, + 'caseruns_allowed': 15, + 'flowruns_allowed': 5, + 'nodes_allowed': 4, + 'conditions_allowed': 1, + 'retention_days': 15, + } + return usage + + + + +def get_meta_default(): + meta = { + 'last_usage_reset': datetime.now(tz.utc).strftime('%Y-%m-%d %H:%M:%S.%f'), + 'coupon': { + 'code': '', + 'discount': 0 + } + } + return meta + + + + +def get_account_info_default(): + info = {'survey': []} + return info + + + + +def get_permissions_default(): + permissions = { + 'actions': [ + 'add', 'get', 'update', 'delete' + ], + 'resources': [ + 'site', 'page', 'issue', 'case', 'caserun', + 'flow', 'flowrun', 'test', 'scan', 'schedule', + 'alert', 'secret', 'report', 'process', 'log' + ], + 'sites': [] + } + return permissions + + + + +def get_system_default(): + system = { + 'tasks': [], + } + return system + + + + +def get_nodes_default(): + nodes = [ + { + 'id': '1', + 'position': { + 'x': 0, + 'y': 0 + }, + 'type': 'basic', + 'parentId': None, + 'data': { + 'id': '1', # duplicate for client support + 'position': { # duplicate for client support + 'x': 0, + 'y': 0 + }, + 'parentId': None, # duplicate for client support + 'task_type': None, + 'configs': settings.CONFIGS, + 'conditions': None, + 'start_if': None, + } + }, + ] + return nodes + + + + +def get_edges_default(): + edges = [] + return edges + + + + +def get_license_key(): + license_key = 'cursion-license-' + secrets.token_hex(32) + return license_key + + + + + + class Account(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True) + # phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) ## -> REMOVING !!!! active = models.BooleanField(default=False, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free') code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) + license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True, default=get_license_key) ## -> NEW!!!!! + # max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!! + # max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!! + # max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=1) ## -> REMOVING!!!! + # retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3) ## -> REMOVING!!!! cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0) + interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month') + usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default) slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) + configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default) ## -> NEW!!!!! + meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default) + def __str__(self): return self.user.email @@ -272,8 +419,10 @@ class Member(models.Model): account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin + phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) ## NEW !!! status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + permissions = models.JSONField(serialize=True, null=True, blank=True, default=get_permissions_default) ## NEW !!!!!!!! time_created = models.DateTimeField(default=timezone.now, serialize=True) def __str__(self): @@ -282,14 +431,29 @@ def __str__(self): +class Secret(models.Model): ### NEW !!!! + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + name = models.CharField(max_length=500, serialize=True, null=True, blank=True) + value = models.TextField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.name}' + + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True) + time_crawl_completed = models.DateTimeField(serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): @@ -297,37 +461,61 @@ def __str__(self): + +class Page(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.page_url}' + + + + class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - html = models.TextField(serialize=True, null=True, blank=True) + html = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) + score = models.FloatField(serialize=True, null=True, blank=True) lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) configs = models.JSONField(serialize=True, null=True, blank=True) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + system = models.JSONField(serialize=True, null=True, blank=True, default=get_system_default) def __str__(self): return f'{self.id}__scan' + class Test(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) + threshold = models.FloatField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, null=True, blank=True) component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default) - html_delta = models.JSONField(serialize=True, null=True, blank=True) + html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) @@ -337,48 +525,45 @@ class Test(models.Model): post_scan_configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__test' + return f'{self.id}_test' -class Schedule(models.Model): +class Case(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') - time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) ## RENAMED !!!! from name to title user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase - timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) - begin_date = models.DateTimeField(default=datetime.now, serialize=True) - time = models.CharField(max_length=100, null=True, blank=True, serialize=True) - frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly, - task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task - crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) - periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) - status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) - extras = models.JSONField(serialize=True, null=True, blank=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + processed = models.BooleanField(default=False, serialize=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.site.site_url}__{self.task_type}' + return f'{self.title}' if self.title else str(id) ### REMANED from self.name to self.title -class Automation(models.Model): +class CaseRun(models.Model): ## -> RENAME from Testcase to CaseRun !!!!!!! id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + title = models.CharField(max_length=500, null=True, blank=True, serialize=True) ## RENAMED !!!! from case_name to title + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') - expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) - actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) + time_completed = models.DateTimeField(null=True, blank=True, serialize=True) + status = models.CharField(max_length=20, default='working', null=True, blank=True, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True) + configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.name}' - + return f'{self.title}_caserun' ## RENAMED !!! from case.name to title @@ -386,53 +571,115 @@ def __str__(self): class Report(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab] + type = models.JSONField(serialize=True, null=True, blank=True) info = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__report' + return f'{self.page.page_url}_report' - -class Case(models.Model): +class Issue(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + trigger = models.JSONField(serialize=True, null=True, blank=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + details = models.TextField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, default='open') + affected = models.JSONField(serialize=True, null=True, blank=True) + labels = models.JSONField(serialize=True, null=True, blank=True) + read = models.BooleanField(default=False, serialize=True) + + def __str__(self): + return f'{self.title if self.title is not None else self.id}_issue' + + + + +class Flow(models.Model): ## -> NEW !!!!!!! + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) time_created = models.DateTimeField(default=timezone.now, serialize=True) - steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) - tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + time_last_run = models.DateTimeField(serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + nodes = models.JSONField(serialize=True, null=True, blank=True, default=get_nodes_default) + edges = models.JSONField(serialize=True, null=True, blank=True, default=get_edges_default) def __str__(self): - return f'{self.name}' + return f'{self.title if self.title is not None else self.id}_flow' -class Testcase(models.Model): +class FlowRun(models.Model): ## -> NEW !!!!!!! id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + flow = models.ForeignKey(Flow, on_delete=models.CASCADE, null=True, blank=True, serialize=True) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - time_created = models.DateTimeField(default=timezone.now, serialize=True) - time_completed = models.DateTimeField(null=True, blank=True, serialize=True) - passed = models.BooleanField(default=False, serialize=True) - steps = models.JSONField(serialize=True, null=True, blank=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, default='working') + nodes = models.JSONField(serialize=True, null=True, blank=True) + edges = models.JSONField(serialize=True, null=True, blank=True) + logs = models.JSONField(serialize=True, null=True, blank=True) configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.case.name}__testcase' + return f'{self.flow.title if self.flow.title is not None else self.id}_flowrun' - + + +class Schedule(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + scope = models.CharField(max_length=100, default='account', serialize=True) + resources = models.JSONField(serialize=True, null=True, blank=True) + alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert') ## NEW !!!! + # automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') ## REMOVE !!!! + time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) + time_last_run = models.DateTimeField(null=True, blank=True, serialize=True) + task_type = models.CharField(max_length=100, default='test', serialize=True) + timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) + begin_date = models.DateTimeField(default=datetime.now, serialize=True) + time = models.CharField(max_length=100, null=True, blank=True, serialize=True) + frequency = models.CharField(default="monthly", max_length=100, serialize=True) + task = models.CharField(max_length=500, null=True, blank=True, serialize=True) + crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.account.name}_{self.task_type}' + + + + +class Alert(models.Model): ## -> RENAME from Automation to alert !!!!!!! + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') + expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) + actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) + + def __str__(self): + return f'{self.id}' + @@ -443,7 +690,7 @@ class Mask(models.Model): mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__mask' + return f'{self.id}_mask' @@ -451,15 +698,19 @@ def __str__(self): class Process(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + object_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - successful = models.BooleanField(serialize=True, default=False) + success = models.BooleanField(serialize=True, default=False) + exception = models.TextField(serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True) info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) progress = models.FloatField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__process' + return f'{self.id}_process' @@ -475,4 +726,7 @@ class Log(models.Model): response_payload = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.status}__{self.request_type}__{self.path}' + return f'{self.status}_{self.request_type}_{self.path}' + + + diff --git a/app/api/signals.py b/app/api/signals.py new file mode 100644 index 00000000..d799cdca --- /dev/null +++ b/app/api/signals.py @@ -0,0 +1,26 @@ +from django.db.models.signals import post_save +from django.dispatch import receiver +from .utils.flowr import Flowr +from .models import * +from cursion import settings + + + + + + +@receiver(post_save, sender=FlowRun) +def flowrun_created(sender, instance, created, **kwargs): + + # defing instance as new flowrun + flowrun = instance + + # check location + if settings.LOCATION == 'us': + + # init Flowr & execute run_next() + Flowr(flowrun_id=str(flowrun.id)).run_next() + + # return None + return None + \ No newline at end of file diff --git a/app/api/tasks.py b/app/api/tasks.py index d242b559..466060da 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,204 +1,3196 @@ -from __future__ import absolute_import, unicode_literals -from typing import Any from celery.utils.log import get_task_logger -from celery import shared_task -from celery import Task as BaseTask -from .v1.ops.tasks import ( - create_site_task, create_scan_task, run_html_and_logs_task, - run_vrt_task, run_lighthouse_task, run_yellowlab_task, - create_test_task, create_report_task, delete_report_s3, - delete_site_s3, create_testcase_task, migrate_site_task, - delete_testcase_s3, +from celery import shared_task, Task +from cursion import celery +from .utils.crawler import Crawler +from .utils.scanner import Scanner as S +from .utils.tester import Tester as T +from .utils.reporter import Reporter as R +from .utils.wordpress import Wordpress as W +from .utils.alerter import Alerter +from .utils.caser import Caser +from .utils.autocaser import AutoCaser +from .utils.exporter import create_and_send_report_export +from .utils.scanner import ( + _html_and_logs, _vrt, _lighthouse, + _yellowlab +) +from .utils.alerts import * +from .utils.updater import update_flowrun +from .models import * +from django.contrib.auth.models import User +from django.utils import timezone +from datetime import datetime, timedelta, timezone +from redis import Redis +from contextlib import contextmanager +from cursion import settings +import asyncio, boto3, time, requests, \ +json, stripe, inspect, random, secrets + + + + + + +class BaseTaskWithRetry(Task): + autoretry_for = (Exception, KeyError) + retry_kwargs = {'max_retries': int(settings.MAX_ATTEMPTS - 1)} + retry_backoff = True + + + + +# setting logger +logger = get_task_logger(__name__) + + + + +# setting redis client +redis_client = Redis.from_url(settings.CELERY_BROKER_URL) + + + + +# setting locking manager to prevent duplicate tasks +@contextmanager +def task_lock(lock_name, timeout=300): + lock = redis_client.lock(lock_name, timeout=timeout) + acquired = lock.acquire(blocking=False) + print(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}") + try: + yield acquired + finally: + if acquired: + lock.release() + print(f"Lock released for {lock_name}") + + + + +# setting s3 instance +def s3(): + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + return s3 + + + + +def check_and_increment_resource(account: object, resource: str) -> bool: + """ + Adds 1 to the Account.usage.{resource} if + {resource}_allowed has not been reached. + + Expcets: { + 'account' :

Your migration is complete!