diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e35367f9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,32 @@ +db.sqlite3 +.DS_Store + +*__pycache__* +*.pyc +__pycache__ +__pycache__/ +*/__pycache__/* +**/__pycache__/ + +env/.env.local +env/.env.dev +env/.env.prod +env/.env.stage +env/.env.remote +env/.env.prod.db +env/.env.client.dev +env/.env.client.prod + +app/data* +app/static* +app/chromedriver* +app/.cache* +app/.config* +app/.local* +app/.pki* + +k8s/*/*-configs.yaml +k8s/prod/old_configs/* + +notes/private* + diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml new file mode 100644 index 00000000..d57de616 --- /dev/null +++ b/.github/workflows/dev-deploy.yaml @@ -0,0 +1,35 @@ +name: Deploy to dev + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - dev1 # NOTE -> pausing auto "dev" deployment + paths: + - 'app/**' + - 'Dockerfile' + - 'docker-compose.dev.yml' + - '.github/workflows/**' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: pulling latest dev commit and rebuilding app + uses: appleboy/ssh-action@v1.0.0 + with: + host: ${{ secrets.DEV_HOST }} + username: ${{ secrets.DEV_USER }} + key: ${{ secrets.DEV_KEY }} + command_timeout: 30m + script: | + cd ~/app + git pull origin dev + docker compose -f docker-compose.dev.yml down + docker volume rm app_server app_beat app_celery + docker image rm cursiondev/client + docker compose -f docker-compose.dev.yml up -d --build + docker system prune -f \ No newline at end of file diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml new file mode 100644 index 00000000..7b80e0cc --- /dev/null +++ b/.github/workflows/k8s-deploy.yaml @@ -0,0 +1,84 @@ +# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and +# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, +# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions. + +name: Deploy to cursion-k8s + +# Controls when the action will run. +on: + # Triggers the workflow on push request on the main branch for changes in the specified paths. + push: + branches: + - main + paths: + - 'app/**' + - 'k8s/prod/**' + - 'Dockerfile' + - '.github/workflows/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel. +jobs: + # This workflow contains a single job called "build". + build: + # The type of runner that the job will run on. + runs-on: ubuntu-latest + timeout-minutes: 30 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it. + - name: Checkout main + uses: actions/checkout@main + + # Install doctl. + - name: Install doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + + # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA. + - name: Build container image + run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) -t ${{ secrets.REGISTRY_NAME }}/server:latest . + + - name: Log in to Docker Hub Container Registry with short-lived credentialse + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + + - name: Push unique tag to Docker Hub Container Registry + run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) + + - name: Push latest tag to Docker Hub Container Registry + run: docker image push ${{ secrets.REGISTRY_NAME }}/server:latest + + # Update deployment files to reflect new build. + - name: Update app deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + + - name: Update celery on_demand deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-on-demand-deployment.yaml + + - name: Update celery scheduled deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-scheduled-deployment.yaml + + - name: Update beat deployment file + run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's||${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml + + - name: Save DigitalOcean kubeconfig with short-lived credentials + run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }} + + # Re-deploy app, celery and beat + - name: Deploy app + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml + - name: Deploy celery on_demand + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-on-demand-deployment.yaml + - name: Deploy celery scheduled + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-scheduled-deployment.yaml + - name: Deploy beat + run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml + + - name: Verify app + run: kubectl rollout status deployment/app-deployment + - name: Verify celery + run: kubectl rollout status deployment/celery-scheduled-deployment + - name: Verify beat + run: kubectl rollout status deployment/beat-deployment diff --git a/.gitignore b/.gitignore index 780009eb..26dd46eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,34 @@ -app/api/utils/testing_stuff.py -app/data* -app/api/utils/__pycache__/tester.cpython-38.pyc +db.sqlite3 .DS_Store + *__pycache__* -db.sqlite3 *.pyc __pycache__ __pycache__/ */__pycache__/* **/__pycache__/ -server/app/env* + env/.env.local env/.env.dev env/.env.prod +env/.env.stage +env/.env.remote env/.env.prod.db +env/.env.client.dev +env/.env.client.prod + +app/data* app/static* -Dockerfile.alpine -Dockerfile.dev1 -Dockerfile.dev3 app/api/migrations/*_*.py +app/chromedriver* +app/.cache* +app/.config* +app/.local* +app/.pki* + k8s/*/*-configs.yaml +k8s/prod/old_configs/* + +notes/private* + + diff --git a/Dockerfile b/Dockerfile index c1f55754..27762225 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,119 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 +# pull main python image +FROM python:3.12-slim -# create the app user -RUN addgroup --system app && adduser --system app +# adding labels +LABEL Author="Cursion" Support="hello@cursion.dev" -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip +# setting ENVs and Configs +ENV HOME=/app +ENV XDG_CACHE_HOME=$HOME/.cache +ENV DOCKERIZED=yes +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV MOZ_NO_REMOTE=1 +ENV MOZ_DISABLE_AUTO_SAFE_MODE=1 +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable +ENV PYTHONPATH="$HOME:$PYTHONPATH" +ENV NODE_OPTIONS="--max-old-space-size=4080" +ENV DJANGO_ALLOWED_HOSTS="*" +ENV SECRET_KEY="abcdefghijklmno123456789" -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig +# create the app user +RUN addgroup --system app && adduser --system app -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ - && npm install -g n && n lts +# Clean cache to avoid issues +RUN apt-get clean && rm -rf /var/lib/apt/lists/* -# increasing allocated memory to node -RUN export NODE_OPTIONS="--max-old-space-size=4096" +# installing system deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-client \ + gcc \ + make \ + gfortran \ + openssl \ + libpq-dev \ + curl \ + libsm6 \ + libxrender1 \ + libxext6 \ + libgl1 \ + nasm \ + autoconf \ + libtool \ + automake \ + libjpeg-dev \ + libglib2.0-0 \ + libfreetype6 \ + ca-certificates \ + libfontconfig \ + gnupg + +# installing firefox-esr +RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools +# installing google-chrome-stable +RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get install -y ./google-chrome-stable_current_amd64.deb && \ + rm google-chrome-stable_current_amd64.deb +# installing microsoft-edge-stable +RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \ + install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \ + sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \ + /etc/apt/sources.list.d/microsoft-edge.list' && \ + apt-get update && apt-get install -y microsoft-edge-stable -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +# installing node and npm +RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \ + apt-get install -y --no-install-recommends nodejs && \ + npm install -g --no-cache n && \ + n lts -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes +# installing lighthouse & lighthouse-plugin-crux +RUN npm install -g lighthouse lighthouse-plugin-crux -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version +# installing lodash & yellowlabtools +RUN npm install -g lodash yellowlabtools -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt +# copying & installing requirements +COPY ./setup/requirements/requirements.txt /requirements.txt +RUN python3.12 -m pip install -r /requirements.txt # setting working dir -RUN mkdir /app COPY ./app /app WORKDIR /app +# setting browser cache dirs +RUN mkdir -p .mozilla .cache + # setting ownership RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file +RUN chown -R app:app /usr/local/bin/lighthouse +RUN chown -R app:app /usr/local/bin/yellowlabtools + +# writing migrations file +RUN python3.12 manage.py makemigrations --no-input + +# collecting static assets +RUN python3.12 manage.py collectstatic --no-input + +# cleaning up +RUN apt-get clean && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + microsoft.gpg + +# setting final user +USER app + +# copy healthcheck.sh +COPY ./setup/scripts/healthcheck.sh "/healthcheck.sh" + +# staring up services +COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh" +ENTRYPOINT [ "/entrypoint.sh" ] + + + diff --git a/Dockerfile.prod b/Dockerfile.prod deleted file mode 100644 index eb8edf96..00000000 --- a/Dockerfile.prod +++ /dev/null @@ -1,52 +0,0 @@ -FROM python:3.9-slim -ENV PYTHONUNBUFFERED 1 - -# create the app user -RUN addgroup --system app && adduser --system app - -# installing python3 & pip -RUN apt-get update && apt-get install -y python3 python3-pip - -# installing system deps -RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ - gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ - libfontconfig git - -# installing node and npm -RUN apt-get update && apt-get install nodejs npm -y \ - && npm install -g n && n lts - -RUN npm cache clean --force - -# increasing allocated memory to node -# RUN export NODE_OPTIONS="--max-old-space-size=7000" -# RUN export NODE_OPTIONS="--stack-size=262000" -ENV NODE_OPTIONS=--max_old_space_size=7000 -ENV NODE_OPTIONS="--max-old-space-size=7000" -# ENV NODE_OPTIONS=--stack-size=262000 - -# installing lighthouse & yellowlabtools -RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools - -# telling Puppeteer to skip installing Chrome -ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true - -# telling phantomas where Chromium binary is and that we're in docker -ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium -ENV DOCKERIZED yes - -# setting --no-sandbox for Phantomas -RUN chromium --no-sandbox --version - -# installing requirements -COPY ./requirements.txt /requirements.txt -RUN python3 -m pip install -r /requirements.txt - -# setting working dir -RUN mkdir /app -COPY ./app /app -WORKDIR /app - -# setting ownership -RUN chown -R app:app /app -RUN chown -R app:app /usr/bin/chromium \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index ea474ece..fa89ec11 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,101 +1,8 @@ -Copyright (c) 2023 Scanerr +Copyright (C) 2026 Grey Labs, LLC -Scanerr Commercial Software License Terms +> This software **(Cursion Server)**, is offered with a dual-license depending on your use case. -1. Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order. -2. Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues. -3. Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software. -4. Billing. -(a) Bills, Fees, and Payment. The vendor agrees to bill the customer per the order. The customer agrees to pay the fees on the order, using the payment method on the order. -(b) Billing Errors. The customer agrees to give the vendor notice of any suspected error on a bill before the deadline for payment. Both sides agree to resolve any concerns about bill accuracy promptly and in good faith. The customer agrees to pay the undisputed part of each bill by the original deadline, and any part of the bill resolved later within seven days of resolution. -5. Term and Termination. -(a) Perpetual. This agreement continues until one side or the other ends it. -(b) Termination. Either side can terminate this agreement immediately if the other side breaches and fails to cure their breach within fourteen days of notice. -6. Use. -(a) Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates. -(b) Prohibited Uses. The customer may not: -(i) sell, lease, license, or sublicense the software or documentation -(ii) allow access to the software by others not licensed under this agreement -(iii) share copies of the software or documentation with with others not licensed under this agreement -(iv) make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers -(v) assist or allow others to use the software against the terms of this agreement -7. Licenses. -(a) Software Copyright License. The vendor grants the customer and each authorized user a standard license for any copyrights in the software that the vendor can license, to copy, install, back up, and use the software as allowed under this agreement. -(b) Software Patent License. The vendor grants the customer and each authorized user a standard license for any patents the vendor can license or becomes able to license, to use the software as allowed under this agreement. -(c) Documentation Copyright License. The vendor grants the customer and each authorized user a standard license for any copyrights in the documentation that the vendor can license, to read, back up, and copy the documentation. -(d) Standard License Terms. A standard license means a nonexclusive license for the term of this agreement, for versions of the software covered by this agreement, that is conditional on payment of all fees as required by this agreement and subject to any use limits in this agreement. -(e) No Other Licenses. Apart from the licenses in Section 7 (Licenses), this agreement does not license or assign any intellectual property rights. -8. Open Source. -(a) Open Source Compliance. Some components of the software may be open source software available under free, public licenses. If the public license terms for any open source component conflict with the terms of this agreement, only the public license terms apply to that component, not the terms of this agreement. If the license terms for any open source component require an offer of source code or other information related to that component, the vendor agrees to provide on written request. -(b) Dual Licensing. If any part of the software is or becomes available under a public license: -(i) While the customer’s licenses continue, the customer and each authorized user must abide by this agreement, not the public license. -(ii) The customer must abide by the terms of the public license for any versions of the software not covered by this agreement. -9. Delivery. -(a) Materials. The vendor agrees to deliver the following to the customer within three days: -(i) a copy of the software’s source code in the preferred form for making changes -(ii) copies of any scripts or configuration files necessary to compile the software’s source code -(iii) a copy of the software’s documentation -(b) Method. The vendor agrees to deliver all materials by e-mail or by making them available to download online, without any additional charge. The vendor agrees to make new versions of the software covered by this agreement available in the same way, within three days of making it generally available. -(c) License Keys. If the software requires license keys to function, the vendor agrees to give the customer those keys by e-mail within three days. If license keys for the software expire over time, the vendor agrees to give the customer new license keys by e-mail at least two weeks before the last keys expire. The customer agrees to share license keys only as required for use of the software as allowed under this this agreement, and to secure its license keys at least as well as its confidential business information. -10. Technical Support. -(a) Basic Support. During its regular business hours, the vendor agrees to respond to e-mail support requests from customer or any authorized user about configuration of, use of, and problems with the software and its documentation. The vendor does not agree to any specific service levels for response to support requests. -(b) Access. On the vendor’s request, the customer agrees to give the vendor prompt access to personnel, systems, and information needed to respond to support requests. -(c) Confidentiality. On the customer’s request, the vendor will agree to the terms of a standard, published, mutual nondisclosure agreement with the customer, for the purpose of fulfilling its support obligations under this agreement. -11. Warranties. -(a) Perform As Documented. The vendor guarantees that the software will perform as described in its documentation while this agreement continues, except when: -(i) using older version of the software than the latest provided under this agreement -(ii) using the software with modifications -(iii) running the software using hardware or software different from that required, according to the documentation -(iv) combining the software with other software or hardware in ways not described in the documentation -(b) Malware. The vendor guarantees that the software it delivers will be free of malicious code, such as computer worms and viruses. -(c) Limiting Code. The developer guarantees that the software it delivers will be free of code that automatically limits or disables software functionality, other than: -(i) code that limits or disables functionality on failure to validate license keys -(ii) code that limits or disables functionality based on automatic monitoring of agreed limits on usage -(d) Software Dependencies. If the software depends on, installs, configures, or links to other software in order to function, the vendor guarantees that those software dependencies will be either provided in the copies of the software delivered to the customer or generally available for the customer to download, free or charge, from a well known website or Internet service, such as an open source software package repository. -12. Liability. -(a) Disclaimer. Section 11 (Warranties) sets out the only warranties the vendor provides for the software. The vendor disclaims any warranties the law might otherwise imply, like warranties of merchantability, fitness for any particular purpose, title, or noninfringement. -(b) Unforeseeable Damages. Neither side will be liable for breach-of-contract damages they could not have reasonably foreseen when entering into this agreement. -(c) Liability Cap. Except for Section 12(d) (Uncapped Liabilities), neither side’s total liability for breach of this agreement will exceed the amount of fees the vendor received from the customer under this agreement during the twelve months before the first claim is made. This limit applies even if the side liable is advised that the other may suffer damages, and even if the customer paid no fees at all. -(d) Uncapped Liabilities. Section 12(c) (Liability Cap) does not apply to: -(i) the customer’s obligations to pay fees -(ii) the vendor’s obligations to indemnify the customer -(iii) liabilities the law requires to be unlimited -13. Indemnities. These indemnities apply as long as the customer has paid all licensing fees as required by this agreement: -(a) General Indemnity. Subject to Section 13(e) (Indemnification Process), the vendor agrees to indemnify the customer for legal claims by others alleging that the software infringes any copyright, trademark, or trade secret right, or breaks any law. -(b) Patent Indemnity. The vendor will not indemnify the customer for any claims by others alleging that the software infringes any patent. -(c) Scope of Indemnity. Throughout this agreement, to indemnify means to indemnify and hold the customer and its personnel harmless for all liability, expenses, damages, and costs, as well as to defend the indemnified party. -(d) Only Remedy. Both sides agree that indemnification will be the only legal remedy for claims covered by indemnity. -(e) Indemnification Process. Both sides agree that to receive indemnification under this agreement, they must give notice of any covered claim quickly, allow the other side to control investigation, defense, and settlement, and cooperate with those efforts. Both sides agree that if they fail to give notice of any covered claim quickly, indemnification will not cover amounts that could have been defended against or mitigated if notice had been given quickly. Both sides agree that if they take control of the defense and settlement of any covered claim, they will not agree to any settlements that admit fault or impose obligations on the other side without their signed, written permission. -(f) Repair, Replace, Refund. If the vendor or the customer receives written notice of a claim that the software infringes any intellectual property right or breaks any law, or vendor reasonably anticipates a claim of that kind: -(i) The developer may provide the customer a new version of the software that no longer infringes or breaks the law. That new version will be covered by this agreement. The customer will not pay any additional fee for the new version. -(ii) If the problem is infringement, the developer may get licenses for the customer so that the customer’s use of the software no longer infringes. -(iii) If the problem is illegality, the developer may get the approvals, licenses, or other requirements needed to abide by the law. -(iv) The developer may refund any fees the customer has prepaid under this agreement for time remaining in the term of this agreement, on a proportional basis, and end this agreement immediately by giving the customer notice. -14. Tax. -(a) Taxes on Fees. The customer agrees to pay all tax on fees under this agreement, except tax on the vendor’s income. -(b) Tax Withholding. If the customer is located outside the United States and local law requires the customer to withhold taxes on fees paid under this agreement: -(i) The customer agrees to make the required tax withholding payments for the vendor by deducting the right amounts from payments to the vendor and paying them to the proper tax authorities. -(ii) The customer agrees to increase the amount of each payment made under this agreement, to offset withholding, so that the vendor receives the full amount owed. -(iii) The customer agrees to give the vendor relevant official tax documentation and tax receipts showing that withholding was required and that proper withholding payment was made, as soon as possible after making any withholding payment. -15. General Contract Terms. -(a) Notices. Both sides agree to give notice under this agreement, the side giving notice must send by e-mail to the address the recipient gave with its signature, or to a different address given later for notices going forward, in the English language. If either side finds that e-mail can’t be delivered to the e-mail address given, the sender may give notice by registered mail to the address on file for the recipient with the state under whose laws it is organized. -(b) Governing Law. This agreement will be governed by the law of the jurisdiction of the address the vendor gives with its signature. -(c) No CISG. The United Nations Convention on Contracts for the International Sale of Goods will not apply to this agreement. -(d) No UCITA. As far as the law allows, the Uniform Computer Information Transactions Act will not apply to this agreement. -(e) Dispute Resolution. Any dispute, controversy or claim arising out of or relating to this contract, including the formation, interpretation, breach or termination thereof, including whether the claims asserted are arbitrable, will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The Tribunal will consist of one arbitrator. The place of arbitration will be the capital of the jurisdiction whose laws govern this agreement. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator(s) may be entered in any court having jurisdiction thereof. -(f) Enforcement. Only the parties may enforce rights under this agreement. -(g) Forum for Disputes. Both sides agree to bring any lawsuits related to this agreement in courts in the capital of the jurisdiction whose laws govern this agreement. Both sides consent to the exclusive jurisdiction of those courts and waive any objection that they would be an inconvenient forum for a lawsuit. Both sides agree that the other side can enforce judgments from those courts in other jurisdictions. -(h) Only Terms. Both sides intend the terms of this agreement, together with the order, as the final, complete, and only expression of their agreement about the software. -(i) Unenforceable Terms. If a court decides that any part of this agreement is invalid or unenforceable for any reason, and that enforcing the rest of this agreement would not defeat the purpose of this agreement, then rest of this agreement will still apply. -(j) Excuses. Neither side will be liable for any failure or delay meeting any obligation under this agreement caused by: -(i) failure of the other side or its personnel to meet their obligations under this agreement -(ii) actions done or delayed at the written request of the other side -(iii) fire, flood, earthquake, and other natural disasters -(iv) declared and undeclared wars, acts of terrorism, sabotage, riots, civil disorder, rebellions, and revolutions -(v) extraordinary malfunction of Internet infrastructure, data centers, or communication utilities -(vi) government actions taken in response to any of these causes -(k) Amendments. Both sides may change or add to the terms of this agreement only by signing a written amendment. -(l) Waivers. Both sides will waive terms of this agreement, if at all, only in signed writing. -(m) No Assignment. Neither side may assign any right under this agreement without the other side’s signed, written permission. Neither side will unreasonably refuse permission. Any attempt to assign against the terms of this agreement will have no legal effect. -(n) No Delegation. Neither side may delegate any performance under this agreement. Any attempt to delegate will have no legal effect. +1. **[Open Source GPL v3](legal/OSS.md)**: Developers and individuals who wish to use this license must adhear to all requirments in the GPL v3 license listed in `legal/OSS.md`. If those requirements are unsatisfactory for your use case, please consider a commercial license. + +2. **[Commercial](legal/COMMERCIAL.md)**: Entities who wish to utilise Cursion in a commercial setting and who require additional support can purchase a commercial license via the self-serve [billing](https://app.cursion.dev/billing) or by speaking to our [sales team](https://cursion.dev/booking) \ No newline at end of file diff --git a/README.md b/README.md index d7c28503..9d3f41a4 100644 --- a/README.md +++ b/README.md @@ -1,146 +1,96 @@ -# Scanerr Server (API repo) - -[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io) - -This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. - -> This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information. - - -Copyright © Scanerr 2023 - ---- -  - -## Table of Contents -  - -#### Env's and deployment -- [Scanerr Server (API repo)](#scanerr-server-api-repo) - - [Table of Contents](#table-of-contents) - - [Env's and deployment](#envs-and-deployment) - - [Environment](#environment) - - [Local](#local) - - [Remote](#remote) - - [Scripts](#scripts) - - -  - ---- -  - -## Environment - -Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files. **Never store actual .env's in a repo.** Things to change: -- high level django configs -- admin credentials -- email credentials -- database configs -- google API keys -- stripe keys -- OAuth keys -- twilio credentials -- slack credentials -- s3 remote storage credentials - -  - ---- -  - -## Local -Install and run locally on your machine in a dev environment. - -> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step. - -```shell -$ pip3 install virtualenv -$ virtualenv appenv -$ source appenv/bin/activate -$ mkdir app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose up --build -``` -*Spin-down the application* -```shell -$ docker compose up down -``` - -  - ---- -  - -## Remote -Install and deploy remotely in a production environment. - -> Ensure you have Docker installed and running on your server prior to this step. - -*Server configurations for Ubuntu 20.04* -``` shell -$ ssh root@your_server_ip -# apt update -# apt upgrade -# adduser {user} -# usermod -aG sudo {user} -# ufw allow OpenSSH -# ufw enable -# su {user} -``` - -*Add user to docker group* -```shell -$ sudo usermod -aG docker {user} -$ newgrp docker -``` - -*Generate SSH keys for GitHub* -``` shell -$ ssh-keygen -t ed25519 -C "your_github_email@example.com" -``` -- press `Enter` 3 times -```shell -$ eval "$(ssh-agent -s)" -$ ssh-add ~/.ssh/id_ed25519 -$ cat ~/.ssh/id_ed25519.pub -``` -- copy key to clipboard and paste in GutHub - - -*Create a dir to clone the app into* -``` shell -$ cd ~ -$ mkdir app -$ cd app -$ git clone https://github.com/Scanerr-io/server.git -``` -*Spin-up the application* -```shell -$ docker compose -f docker-compose.prod.yml up -d --build -``` -*Spin-down the application* -```shell -$ docker compose -f docker-compose.prod.yml down -``` -*Spin-down the application and removes the volumes* -```shell -$ docker-compose -f docker-compose.prod.yml down -v -``` - - -  - ---- - -  - -## Scripts - -*ssh into container* -``` shell -$ docker exec -it /bin/sh -``` +

+ + Cursion + +

+ +

+ + + + + + + + GitHub Actions Workflow Status + +

+ +

+ Complicated Web Testing on Easy Mode +

+ +

+ Documentation   |    + CLI   |    + Selfhost   |    + + Join Slack + +

+ +
+
+ +# ✨ Welcome to Cursion +API-first, open-source, and beginner friendly. Cursion is built for the busy developer, automating functional, performance, and structural testing in one platform. + +
+
+ + +# 🛒 What's Included +- [x] Advanced, 3-Step Visual Regression Testing +- [x] Page Source Regression Testing +- [x] [Lighthouse](https://github.com/GoogleChrome/lighthouse) Performance Testing +- [x] [YellowLab](https://github.com/YellowLabTools/YellowLabTools) Performance Testing +- [x] Selenium-based Functional Testing +- [x] Functional Test Generator +- [x] A.I. Generated Issues (via OpenAI) +- [x] A.I. VRT Analysis & Review (via OpenAI) + +
+
+ + +# 💻 Installation Guides +- [Docker Guide](notes/Docker.md) +- [Kubernetes Guide](notes/Kubernetes.md) + +
+
+ +# 🛠️ Contributions +This will be a work-in-progress. For feature ideas and bug fixes please refernce best practices found here: https://opensource.guide/how-to-contribute/ + +
+
+ +# 🙏 Acknowledgements +Special thanks to [@ashrafsamhouri](https://github.com/ashrafsamhouri) with [@activepieces](https://github.com/activepieces) and [@basilakis](https://github.com/Basilakis) for the open source inspiration. + +
+
+ +Copyright © 2026 Grey Labs, LLC + diff --git a/app/api/admin.py b/app/api/admin.py index 16e08a0b..6ef32941 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,51 +1,172 @@ from django.contrib import admin from .models import * from datetime import datetime +from .v1.ops.services import ( + create_scan, create_test, + delete_site, delete_page, + delete_scan, delete_test, + delete_case, delete_caserun, + crawl_site, case_pre_run, +) +from .tasks import ( + reset_account_usage, + update_scan_score +) -@admin.register(Site) -class SiteAdmin(admin.ModelAdmin): - list_display = ('site_url', 'user', 'time_created') - search_fields = ('site_url',) -@admin.register(Test) -class TestAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed', 'type') - search_fields = ('site',) -@admin.register(Scan) -class ScanAdmin(admin.ModelAdmin): - list_display = ('id', 'site', 'time_created', 'time_completed') - search_fields = ('site',) - actions = ['mark_as_completed',] +@admin.register(Account) +class AccountAdmin(admin.ModelAdmin): + list_display = ('name', 'user', 'time_created', 'type') + search_fields = ('name', 'user__email') + actions = ['reset_usage',] - def mark_as_completed(self, request, queryset): - queryset.update(time_completed=datetime.now()) + def reset_usage(self, request, queryset): + for account in queryset: + reset_account_usage.delay( + account_id=account.id + ) -@admin.register(Account) -class AccountAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'type') - search_fields = ('__str__',) @admin.register(Member) class MemberAdmin(admin.ModelAdmin): - list_display = ('user', 'account', 'time_created', 'type', 'status') + list_display = ('email', 'account', 'time_created', 'type', 'status') search_fields = ('user__username', 'account__name') + + @admin.register(Card) class CardAdmin(admin.ModelAdmin): list_display = ('__str__', 'brand', 'last_four') search_fields = ('last_four',) + + +@admin.register(Site) +class SiteAdmin(admin.ModelAdmin): + list_display = ('site_url', 'account', 'time_created') + search_fields = ('site_url', 'account__name') + actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites'] + raw_id_fields = ('account', 'user',) + + def crawl_sites(self, request, queryset): + for site in queryset: + crawl_site( + id=site.id, + user=site.account.user + ) + + def scan_sites(self, request, queryset): + for site in queryset: + create_scan( + site_id=site.id, + user_id=site.account.user.id + ) + + def test_sites(self, request, queryset): + for site in queryset: + create_test( + site_id=site.id, + user_id=site.account.user.id + ) + + def delete_sites(self, request, queryset): + for site in queryset: + delete_site( + id=site.id, + user=site.account.user + ) + + + + +@admin.register(Page) +class PageAdmin(admin.ModelAdmin): + list_display = ('page_url', 'account', 'time_created') + search_fields = ('page_url', 'account__name') + actions = ['scan_pages', 'test_pages', 'delete_pages',] + raw_id_fields = ('account', 'user', 'site') + + def scan_pages(self, request, queryset): + for page in queryset: + create_scan( + page_id=page.id, + user_id=page.account.user.id + ) + + def test_pages(self, request, queryset): + for page in queryset: + create_test( + page_id=page.id, + user_id=page.account.user.id + ) + + def delete_pages(self, request, queryset): + for page in queryset: + delete_page( + id=page.id, + user=page.user + ) + + + + +@admin.register(Scan) +class ScanAdmin(admin.ModelAdmin): + list_display = ('id', 'page', 'time_created', 'time_completed') + search_fields = ('page__page_url',) + actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] + raw_id_fields = ('site', 'page', 'paired_scan',) + + def delete_scans(self, request, queryset): + for scan in queryset: + delete_scan( + id=scan.id, + user=scan.page.account.user + ) + + def add_scan_score(self, request, queryset): + for scan in queryset: + update_scan_score.delay( + scan_id=scan.id + ) + + def mark_as_completed(self, request, queryset): + queryset.update(time_completed=datetime.now()) + + + + +@admin.register(Test) +class TestAdmin(admin.ModelAdmin): + list_display = ('id', 'page', 'time_created', 'time_completed', 'status', 'score') + search_fields = ('page__page_url',) + actions = ['delete_tests',] + raw_id_fields = ('site', 'page', 'pre_scan', 'post_scan',) + + def delete_tests(self, request, queryset): + for test in queryset: + delete_test( + id=test.id, + user=test.page.account.user + ) + + + + @admin.register(Report) class ReportAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'user') + raw_id_fields = ('site', 'page', 'user', 'account',) + + @admin.register(Log) @@ -53,29 +174,121 @@ class LogAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'status', 'user') + + +@admin.register(Chat) +class ChatAdmin(admin.ModelAdmin): + list_display = ('__str__', 'time_created', 'status', 'user') + + + + @admin.register(Schedule) class ScheduleAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'status', 'user') + list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created') + raw_id_fields = ('user', 'account',) + -@admin.register(Automation) -class AutomationAdmin(admin.ModelAdmin): + +@admin.register(Alert) +class AlertAdmin(admin.ModelAdmin): list_display = ('__str__', 'time_created', 'schedule', 'user') + raw_id_fields = ('user', 'account',) + + @admin.register(Process) class ProcessAdmin(admin.ModelAdmin): - list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'successful') + list_display = ('__str__', 'time_created', 'time_completed', 'progress', 'success') + + @admin.register(Case) class CaseAdmin(admin.ModelAdmin): - list_display = ('__str__', 'user', 'time_created',) + list_display = ('title', 'user', 'site', 'time_created',) + search_fields = ('title', 'site__site_url') + actions = ['delete_cases', 'start_pre_run'] + raw_id_fields = ('user', 'account', 'site') + + def delete_cases(self, request, queryset): + for case in queryset: + delete_case( + id=case.id, + user=case.user + ) + + def start_pre_run(self, request, queryset): + for case in queryset: + case_pre_run(**{ + 'case_id': str(case.id), + 'user_id': str(case.user.id) + }) + + + + +@admin.register(CaseRun) +class CaseRunAdmin(admin.ModelAdmin): + list_display = ('title', 'user', 'time_created', 'time_completed',) + search_fields = ('title', 'site__site_url') + raw_id_fields = ('user', 'account', 'site', 'case') + + actions = ['delete_caseruns',] + + def delete_caseruns(self, request, queryset): + for caserun in queryset: + delete_caserun( + id=caserun.id, + user=caserun.user + ) + + + + +@admin.register(Issue) +class IssueAdmin(admin.ModelAdmin): + list_display = ('title', 'account', 'time_created', 'status',) + search_fields = ('title', 'affected') + raw_id_fields = ('account',) + + + + +@admin.register(Flow) +class FlowAdmin(admin.ModelAdmin): + list_display = ('title', 'account', 'time_created',) + search_fields = ('title',) + raw_id_fields = ('user', 'account',) + + + + +@admin.register(FlowRun) +class FlowRunAdmin(admin.ModelAdmin): + list_display = ('title', 'account', 'site', 'time_created', 'time_completed', 'status') + search_fields = ('title', 'site__site_url',) + raw_id_fields = ('user', 'account', 'site', 'flow') + + + + +@admin.register(Secret) +class SecretAdmin(admin.ModelAdmin): + list_display = ('__str__', 'account', 'time_created',) + raw_id_fields = ('user', 'account',) + + + + +@admin.register(Coupon) +class CouponAdmin(admin.ModelAdmin): + list_display = ('__str__', 'discount', 'time_created', 'status',) + search_fields = ('code',) -@admin.register(Testcase) -class TestcaseAdmin(admin.ModelAdmin): - list_display = ('__str__', 'user', 'time_created', 'time_completed',) @admin.register(Mask) @@ -88,4 +301,8 @@ def mark_as_inactive(self, request, queryset): queryset.update(active=False) def mark_as_active(self, request, queryset): - queryset.update(active=True) \ No newline at end of file + queryset.update(active=True) + + + + \ No newline at end of file diff --git a/app/api/apps.py b/app/api/apps.py index 36985c8a..275aba4d 100644 --- a/app/api/apps.py +++ b/app/api/apps.py @@ -2,4 +2,7 @@ class ApiConfig(AppConfig): - name = 'api' \ No newline at end of file + name = 'api' + + def ready(self): + import api.signals \ No newline at end of file diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py index 77c68bd6..af9a1b3d 100644 --- a/app/api/management/commands/create_admin.py +++ b/app/api/management/commands/create_admin.py @@ -1,16 +1,23 @@ from django.core.management.base import BaseCommand from rest_framework.authtoken.models import Token from django.contrib.auth.models import User -from ...models import Account +from ...models import Account, Member, get_permissions_default from ...utils.verify import verify -import os +import os, secrets + + + + + +# creates a new Admin user if None exists class Command(BaseCommand): def handle(self, *args, **options): username = os.environ.get('ADMIN_USER') email = os.environ.get('ADMIN_EMAIL') password = os.environ.get('ADMIN_PASS') + mode = os.environ.get('MODE') if User.objects.filter(is_superuser=True).count() == 0: print('Creating Admin User for %s (%s)' % (username, email)) admin = User.objects.create_superuser(email=email, username=username, password=password) @@ -23,15 +30,52 @@ def handle(self, *args, **options): user = User.objects.get(username=username) if not Account.objects.filter(user=user).exists(): print('Funding account for %s' % (username)) - Account.objects.create( + + # default usage + usage = { + 'sites': 0, + 'schedules': 0, + 'scans': 0, + 'tests': 0, + 'caseruns': 0, + 'flowruns': 0, + 'sites_allowed': 1000, + 'pages_allowed': 10, + 'schedules_allowed': 50, + 'scans_allowed': 100000, + 'tests_allowed': 100000, + 'caseruns_allowed': 100000, + 'flowruns_allowed': 100000, + 'nodes_allowed': 50, + 'conditions_allowed': 25, + 'retention_days': 1000, + } + + code = secrets.token_urlsafe(16) + + account = Account.objects.create( + name='Admin', user=user, active=True, - type='enterprise', - max_sites=10000, + type='selfhost' if mode == 'selfhost' else 'admin', + usage=usage, + code=code, + ) + + # get permissonions or default + permissions = get_permissions_default() + + member = Member.objects.create( + user=user, + email=email, + status='active', + type='admin', + account=account, + permissions=permissions ) + else: print('Accounts can only be initialized if no Accounts exist') if not Token.objects.filter(user=user).exists(): - Token.objects.create(user=user) - # verify() \ No newline at end of file + Token.objects.create(user=user) \ No newline at end of file diff --git a/app/api/management/commands/create_tasks.py b/app/api/management/commands/create_tasks.py new file mode 100644 index 00000000..71f35fef --- /dev/null +++ b/app/api/management/commands/create_tasks.py @@ -0,0 +1,58 @@ +from django.core.management.base import BaseCommand +from django_celery_beat.models import PeriodicTask, IntervalSchedule +from datetime import datetime + + + + + + +# creating default system tasks +class Command(BaseCommand): + + def handle(self, *args, **options): + + tasks = [ + { + 'every': 10, + 'period': IntervalSchedule.MINUTES, + 'name': 'Redeliver Failed Tasks', + 'task': 'api.tasks.redeliver_failed_tasks' + }, + { + 'every': 1, + 'period': IntervalSchedule.DAYS, + 'name': 'Data Retention Cleanup', + 'task': 'api.tasks.data_retention' + }, + { + 'every': 12, + 'period': IntervalSchedule.HOURS, + 'name': 'Reset Account Usage', + 'task': 'api.tasks.reset_account_usage' + }, + ] + + # loop through and create + # PeriodicTasks for each + for task in tasks: + + print(f'Setting up Task: {task.get('name')}') + + try: + # create the schedule + schedule, created = IntervalSchedule.objects.get_or_create( + every=task.get('every'), + period=task.get('period'), + ) + + # create the task + PeriodicTask.objects.create( + interval=schedule, + name=task.get('name'), + task=task.get('task') + ) + + except Exception as e: + print(e) + diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py deleted file mode 100644 index c85d2705..00000000 --- a/app/api/management/commands/driver_p_test.py +++ /dev/null @@ -1,14 +0,0 @@ -from ...utils.driver_p import driver_test -from django.core.management.base import BaseCommand -import asyncio - -# testing puppeteer, pyppeteer, and chromium installation and configs - -class Command(BaseCommand): - - def handle(self, *args, **options): - asyncio.run(driver_test()) - - - - diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py new file mode 100644 index 00000000..1cadfa8c --- /dev/null +++ b/app/api/management/commands/terminate_worker.py @@ -0,0 +1,74 @@ +from cursion import celery +from django.core.management.base import BaseCommand +import time, os + + + + + + +# init warm shutdown (prevent new task acceptance) +class Command(BaseCommand): + + def handle(self, *args, **options): + + # get worker / pod name + default_worker = 'cursion-celery' + if os.environ.get('THIS_POD_NAME'): + default_worker = str(os.environ.get('THIS_POD_NAME')) + + # get celery worker + this_worker = f"celery@{default_worker}" + + # sending initial SIGTERM to celery worker for warm-shutdown + celery.app.control.broadcast('shutdown', destination=[this_worker]) + + + + +# check if current tasks have completed +def wait_for_tasks_to_complete(): + + # get worker / pod name + default_worker = 'cursion-celery' + if os.environ.get('THIS_POD_NAME'): + default_worker = str(os.environ.get('THIS_POD_NAME')) + + # get celery worker + this_worker = f"celery@{default_worker}" + + def get_task_list(): + + # set default + tasks = 0 + + try: + # Inspect all nodes. + i = celery.app.control.inspect() + + # Tasks received, but are still waiting to be executed. + reserved = i.reserved()[this_worker] + print(f'Reserved tasks -> {str(reserved)}') + + # Active tasks + active = i.active()[this_worker] + print(f'Active tasks -> {str(reserved)}') + + # Sum all tasks + tasks = len(active) + len(reserved) + + except Exception as e: + print(e) + + # return tasks count + return int(tasks) + + # get length of active and reserved task lists + tasks = get_task_list() + + # waiting for tasks to complete + while tasks > 0: + time.sleep(10) + tasks = get_task_list() + + \ No newline at end of file diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/test_driver.py similarity index 83% rename from app/api/management/commands/driver_s_test.py rename to app/api/management/commands/test_driver.py index 3ae6438a..1fe02b82 100644 --- a/app/api/management/commands/driver_s_test.py +++ b/app/api/management/commands/test_driver.py @@ -1,8 +1,12 @@ -from ...utils.driver_s import driver_test +from ...utils.driver import driver_test from django.core.management.base import BaseCommand -# testing selenium, chromedriver, and chromium installation and configs + + + + +# testing selenium, chromedriver, and chromium installation and configs class Command(BaseCommand): def handle(self, *args, **options): diff --git a/app/api/management/commands/verify_account.py b/app/api/management/commands/verify_account.py new file mode 100644 index 00000000..a93de01b --- /dev/null +++ b/app/api/management/commands/verify_account.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand +from ...utils.verify import verify + + + + + + +# verifies deployment +class Command(BaseCommand): + + def handle(self, *args, **options): + verify() \ No newline at end of file diff --git a/app/api/management/commands/wait_for_db.py b/app/api/management/commands/wait_for_db.py index c5edb897..def8f534 100644 --- a/app/api/management/commands/wait_for_db.py +++ b/app/api/management/commands/wait_for_db.py @@ -3,8 +3,13 @@ from django.db.utils import OperationalError from django.core.management import BaseCommand + + + + + +# Django command to pause execution until db is available class Command(BaseCommand): - """Django command to pause execution until db is available""" def handle(self, *args, **options): self.stdout.write('Waiting for database...') diff --git a/app/api/migrations/__init__.py b/app/api/migrations/__init__.py old mode 100644 new mode 100755 diff --git a/app/api/models.py b/app/api/models.py index 72e5b54d..bf7f7315 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -1,11 +1,12 @@ from django.db import models -from django.db import models from django.utils import timezone -from django.urls import reverse from django.contrib.auth.models import User -from datetime import datetime -from django.contrib.postgres.fields import JSONField -import uuid +from cursion import settings +import uuid, secrets + + + + def get_info_default(): @@ -14,17 +15,18 @@ def get_info_default(): 'id': None, 'time_created': None, 'time_completed': None, + 'score': None, }, 'latest_test': { 'id': None, 'time_created': None, 'time_completed': None, - 'score': None + 'score': None, + 'status': None }, 'lighthouse': { 'average': None, 'seo': None, - 'pwa': None, 'crux': None, 'performance': None, 'accessibility': None, @@ -33,7 +35,7 @@ def get_info_default(): 'yellowlab': { 'globalScore': None, 'pageWeight': None, - 'requests': None, + 'images': None, 'domComplexity': None, 'javascriptComplexity': None, 'badJavascript': None, @@ -42,17 +44,34 @@ def get_info_default(): 'badCSS': None, 'fonts': None, 'serverConfig': None, - }, - 'status': { - 'health': None, - 'badge': 'neutral', + } + } + return info_default + + + + +def get_small_info_default(): + info_default = { + 'latest_scan': { + 'id': None, + 'time_created': None, + 'time_completed': None, 'score': None, }, + 'latest_test': { + 'id': None, + 'time_created': None, + 'time_completed': None, + 'score': None, + 'status': None + } } return info_default + def get_lh_delta_default(): lh_delta_default = { "scores": { @@ -60,22 +79,23 @@ def get_lh_delta_default(): "performance_delta": None, "accessibility_delta": None, "best-practices_delta": None, - "pwa_delta": None, "crux_delta": None, "average_delta" : None, "current_average": None, }, + "audits": None } return lh_delta_default + def get_yl_delta_default(): yl_delta_default = { "scores": { "average_delta": None, "pageWeight_delta": None, - "requests_delta": None, + "images_delta": None, "domComplexity_delta": None, "javascriptComplexity_delta": None, "badJavascript_delta": None, @@ -85,11 +105,13 @@ def get_yl_delta_default(): "fonts_delta": None, "serverConfig_delta": None, }, + "audits": None } return yl_delta_default + def get_lh_default(): lh_default = { "scores": { @@ -97,29 +119,22 @@ def get_lh_default(): "performance": None, "accessibility": None, "best_practices": None, - "pwa": None, "crux": None, "average": None }, - "audits": { - "seo": [], - "performance": [], - "accessibility": [], - "best-practices": [], - "pwa": [], - "crux": [] - }, + "audits": None, } return lh_default + def get_yl_default(): yl_default = { "scores": { "globalScore": None, "pageWeight": None, - "requests": None, + "images": None, "domComplexity": None, "javascriptComplexity": None, "badJavascript": None, @@ -129,23 +144,13 @@ def get_yl_default(): "fonts": None, "serverConfig": None, }, - "audits": { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - }, + "audits": None, } return yl_default + def get_expressions_default(): expressions_default = { 'list': [ @@ -161,6 +166,7 @@ def get_expressions_default(): + def get_actions_default(): actions_default = { 'list': [ @@ -199,6 +205,7 @@ def get_steps_default(): + def get_scores_default(): scores_default = { 'html': None, @@ -211,6 +218,7 @@ def get_scores_default(): + def get_slack_default(): slack_default = { "slack_name": None, @@ -223,6 +231,8 @@ def get_slack_default(): return slack_default + + def get_tags_default(): tags_default = None, return tags_default @@ -230,6 +240,137 @@ def get_tags_default(): +def get_default_configs(): + configs = settings.CONFIGS + return configs + + + + +def get_usage_default(): + usage = { + 'sites': 0, + 'schedules': 0, + 'scans': 0, + 'tests': 0, + 'caseruns': 0, + 'flowruns': 0, + 'concurrency': 2, + 'sites_allowed': 1, + 'pages_allowed': 3, + 'schedules_allowed': 1, + 'scans_allowed': 30, + 'tests_allowed': 30, + 'caseruns_allowed': 15, + 'flowruns_allowed': 5, + 'nodes_allowed': 4, + 'conditions_allowed': 1, + 'retention_days': 15, + } + return usage + + + + +def get_meta_default(): + meta = { + 'last_usage_reset': str(timezone.now()), + 'coupon': { + 'code': '', + 'discount': 0 + } + } + return meta + + + + +def get_account_info_default(): + info = {'survey': []} + return info + + + + +def get_permissions_default(): + permissions = { + 'actions': [ + 'add', 'get', 'update', 'delete' + ], + 'resources': [ + 'site', 'page', 'issue', 'case', 'caserun', + 'flow', 'flowrun', 'test', 'scan', 'schedule', + 'alert', 'secret', 'report', 'process', 'log', + 'chat' + ], + 'sites': [] + } + return permissions + + + + +def get_system_default(): + system = { + 'tasks': [], + } + return system + + + + +def get_nodes_default(): + nodes = [ + { + 'id': '1', + 'position': { + 'x': 0, + 'y': 0 + }, + 'type': 'basic', + 'parentId': None, + 'data': { + 'id': '1', # duplicate for client support + 'position': { # duplicate for client support + 'x': 0, + 'y': 0 + }, + 'parentId': None, # duplicate for client support + 'task_type': None, + 'configs': settings.CONFIGS, + 'conditions': None, + 'start_if': None, + } + }, + ] + return nodes + + + + +def get_edges_default(): + edges = [] + return edges + + + + +def get_messages_default(): + messages = [] + return messages + + + + +def get_license_key(): + license_key = 'cursion-license-' + secrets.token_hex(32) + return license_key + + + + + + class Account(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) @@ -238,13 +379,19 @@ class Account(models.Model): time_created = models.DateTimeField(default=timezone.now, serialize=True) type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free') code = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1) + license_key = models.CharField(max_length=100, serialize=True, null=True, blank=True, default=get_license_key) cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0) + interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month') + usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default) slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default) - + configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_account_info_default) + meta = models.JSONField(serialize=True, null=True, blank=True, default=get_meta_default) + def __str__(self): return self.user.email @@ -272,8 +419,10 @@ class Member(models.Model): account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) email = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # created by Account admin - status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + phone = models.CharField(max_length=50, serialize=True, null=True, blank=True) + status = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # pending, active + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # admin, contributor, client + permissions = models.JSONField(serialize=True, null=True, blank=True, default=get_permissions_default) time_created = models.DateTimeField(default=timezone.now, serialize=True) def __str__(self): @@ -282,14 +431,29 @@ def __str__(self): +class Secret(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + name = models.CharField(max_length=500, serialize=True, null=True, blank=True) + value = models.TextField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.name}' + + + class Site(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True) + time_crawl_completed = models.DateTimeField(serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): @@ -297,37 +461,61 @@ def __str__(self): + +class Page(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + + def __str__(self): + return f'{self.page_url}' + + + + class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - html = models.TextField(serialize=True, null=True, blank=True) + html = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs = models.JSONField(serialize=True, null=True, blank=True) images = models.JSONField(serialize=True, null=True, blank=True) + score = models.FloatField(serialize=True, null=True, blank=True) lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default) yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default) configs = models.JSONField(serialize=True, null=True, blank=True) tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + system = models.JSONField(serialize=True, null=True, blank=True, default=get_system_default) def __str__(self): return f'{self.id}__scan' + class Test(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) type = models.JSONField(serialize=True, null=True, blank=True) pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan') post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan') score = models.FloatField(serialize=True, null=True, blank=True) + threshold = models.FloatField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, null=True, blank=True) component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default) - html_delta = models.JSONField(serialize=True, null=True, blank=True) + html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True) logs_delta = models.JSONField(serialize=True, null=True, blank=True) lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default) yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default) @@ -335,50 +523,48 @@ class Test(models.Model): tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) pre_scan_configs = models.JSONField(serialize=True, null=True, blank=True) post_scan_configs = models.JSONField(serialize=True, null=True, blank=True) + system = models.JSONField(serialize=True, null=True, blank=True, default=get_system_default) def __str__(self): - return f'{self.id}__test' + return f'{self.id}_test' -class Schedule(models.Model): +class Case(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto') - time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase - timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) - begin_date = models.DateTimeField(default=datetime.now, serialize=True) - time = models.CharField(max_length=100, null=True, blank=True, serialize=True) - frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly, - task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task - crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) - periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) - status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) - extras = models.JSONField(serialize=True, null=True, blank=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + processed = models.BooleanField(default=False, serialize=True) + tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) def __str__(self): - return f'{self.site.site_url}__{self.task_type}' + return f'{self.title}' if self.title else str(id) -class Automation(models.Model): +class CaseRun(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + title = models.CharField(max_length=500, null=True, blank=True, serialize=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') - expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) - actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) + time_completed = models.DateTimeField(null=True, blank=True, serialize=True) + status = models.CharField(max_length=20, default='working', null=True, blank=True, serialize=True) + steps = models.JSONField(serialize=True, null=True, blank=True) + configs = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.name}' - + return f'{self.title}_caserun' @@ -386,53 +572,134 @@ def __str__(self): class Report(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True) ### REMOVE user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) path = models.CharField(max_length=1000, serialize=True, null=True, blank=True) - type = models.JSONField(serialize=True, null=True, blank=True) # array of [lighthouse, yellowlab] + type = models.JSONField(serialize=True, null=True, blank=True) info = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.site.site_url}__report' + if self.page and self.page.page_url: + return f'{self.page.page_url}_report' + if self.site and self.site.site_url: + return f'{self.site.site_url}_report' + return f'{self.id}_report' + +class Issue(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + trigger = models.JSONField(serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + details = models.TextField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, default='open') + affected = models.JSONField(serialize=True, null=True, blank=True) + labels = models.JSONField(serialize=True, null=True, blank=True) + read = models.BooleanField(default=False, serialize=True) + def __str__(self): + return f'{self.title if self.title is not None else self.id}_issue' -class Case(models.Model): + + + +class Flow(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_last_run = models.DateTimeField(serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + nodes = models.JSONField(serialize=True, null=True, blank=True, default=get_nodes_default) + edges = models.JSONField(serialize=True, null=True, blank=True, default=get_edges_default) + + def __str__(self): + return f'{self.title if self.title is not None else self.id}_flow' + + + + +class FlowRun(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + time_completed = models.DateTimeField(serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + flow = models.ForeignKey(Flow, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + title = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + status = models.CharField(max_length=500, serialize=True, default='working') + nodes = models.JSONField(serialize=True, null=True, blank=True) + edges = models.JSONField(serialize=True, null=True, blank=True) + logs = models.JSONField(serialize=True, null=True, blank=True) + configs = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.flow.title if self.flow.title is not None else self.id}_flowrun' + + + + +class Schedule(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + scope = models.CharField(max_length=100, default='account', serialize=True) + resources = models.JSONField(serialize=True, null=True, blank=True) + tags = models.JSONField(serialize=True, null=True, blank=True) + alert = models.ForeignKey('Alert', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_alert') + time_created = models.DateTimeField(default=timezone.now, null=True, blank=True, serialize=True) + time_last_run = models.DateTimeField(null=True, blank=True, serialize=True) + task_type = models.CharField(max_length=100, default='test', serialize=True) + begin_date = models.DateTimeField(default=timezone.now, serialize=True) + timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True) + time = models.CharField(max_length=100, null=True, blank=True, serialize=True) + frequency = models.CharField(default="monthly", max_length=100, serialize=True) + task = models.CharField(max_length=500, null=True, blank=True, serialize=True) + crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True) + status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True) + extras = models.JSONField(serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.account.name}_{self.task_type}' + + + + +class Alert(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=1000, serialize=True, null=True, blank=True) user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True) account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default) - tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default) + schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch') + expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default) + actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default) def __str__(self): - return f'{self.name}' + return f'{self.id}' -class Testcase(models.Model): +class Chat(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True) - account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) - case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True) - site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) - time_completed = models.DateTimeField(null=True, blank=True, serialize=True) - passed = models.BooleanField(default=False, serialize=True) - steps = models.JSONField(serialize=True, null=True, blank=True) - configs = models.JSONField(serialize=True, null=True, blank=True) + user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True) + status = models.CharField(max_length=100, serialize=True, default='active', null=True, blank=True) + messages = models.JSONField(serialize=True, null=True, blank=True, default=get_messages_default) def __str__(self): - return f'{self.case.name}__testcase' + return f'{self.id}_chat' - @@ -443,7 +710,7 @@ class Mask(models.Model): mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__mask' + return f'{self.id}_mask' @@ -451,15 +718,19 @@ def __str__(self): class Process(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True) - type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True) + type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) + object_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True) time_created = models.DateTimeField(default=timezone.now, serialize=True) time_completed = models.DateTimeField(serialize=True, null=True, blank=True) - successful = models.BooleanField(serialize=True, default=False) + success = models.BooleanField(serialize=True, default=False) + exception = models.TextField(serialize=True, null=True, blank=True) + info = models.JSONField(serialize=True, null=True, blank=True) info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True) progress = models.FloatField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.id}__process' + return f'{self.id}_process' @@ -475,4 +746,18 @@ class Log(models.Model): response_payload = models.JSONField(serialize=True, null=True, blank=True) def __str__(self): - return f'{self.status}__{self.request_type}__{self.path}' + return f'{self.status}_{self.request_type}_{self.path}' + + + + +class Coupon(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + time_created = models.DateTimeField(default=timezone.now, serialize=True) + code = models.CharField(max_length=100, serialize=True, null=True, blank=True) + discount = models.FloatField(serialize=True, null=True, blank=True) + status = models.CharField(max_length=100, serialize=True, null=True, blank=True) + + def __str__(self): + return f'{self.code}' + diff --git a/app/api/queue.py b/app/api/queue.py new file mode 100644 index 00000000..a746c0b5 --- /dev/null +++ b/app/api/queue.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from celery import Task +from celery.utils.log import get_task_logger +from contextlib import contextmanager +from django.apps import apps +from redis import Redis +from cursion import settings +import time, random, secrets + + + + +# setting logger +logger = get_task_logger(__name__) + + +class BaseTaskWithRetry(Task): + autoretry_for = (Exception, KeyError) + retry_kwargs = {'max_retries': int(settings.MAX_ATTEMPTS - 1)} + retry_backoff = True + + +# setting redis client +redis_client = Redis.from_url(settings.CELERY_BROKER_URL) + +CELERY_QUEUE_SCHEDULED = getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled') +CELERY_QUEUE_ON_DEMAND = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + + +def get_task_queue(task_request=None, kwargs: dict | None = None) -> str: + if kwargs and kwargs.get('_queue'): + return str(kwargs['_queue']) + if task_request is not None: + try: + delivery_info = getattr(task_request, 'delivery_info', {}) or {} + routing_key = delivery_info.get('routing_key') + if routing_key: + return str(routing_key) + except Exception: + pass + return str(getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED)) + + +def apply_async_in_queue(task, *, kwargs: dict, queue: str, task_id: str | None = None): + return task.apply_async( + kwargs=kwargs, + queue=queue, + routing_key=queue, + task_id=task_id, + ) + + +_ACCT_SEMAPHORE_SCRIPT = redis_client.register_script( + """ + local held_key = KEYS[1] + local pending_key = KEYS[2] + local token = ARGV[1] + local now_ms = tonumber(ARGV[2]) + local ttl_ms = tonumber(ARGV[3]) + local limit = tonumber(ARGV[4]) + local pending_max_age_ms = tonumber(ARGV[5]) + + redis.call('ZREMRANGEBYSCORE', held_key, '-inf', now_ms) + redis.call('ZREMRANGEBYSCORE', pending_key, '-inf', now_ms - pending_max_age_ms) + + -- ensure token is in pending with enqueue time; don't overwrite if exists + if redis.call('ZSCORE', pending_key, token) == false then + redis.call('ZADD', pending_key, now_ms, token) + end + + local rank = redis.call('ZRANK', pending_key, token) + if rank == false then + rank = 0 + end + + -- FIFO gating: only the first `limit` pending tokens are eligible to run + if rank >= limit then + return {0, rank} + end + + local count = tonumber(redis.call('ZCARD', held_key)) + if count >= limit then + return {0, rank} + end + + redis.call('ZREM', pending_key, token) + redis.call('ZADD', held_key, now_ms + ttl_ms, token) + redis.call('PEXPIRE', held_key, ttl_ms + 60000) + redis.call('PEXPIRE', pending_key, pending_max_age_ms + 60000) + return {1, rank} + """ +) + + +def _account_semaphore_key(account_id: str) -> str: + return f"semaphore:account:{account_id}:held" + + +def _account_pending_key(account_id: str) -> str: + return f"semaphore:account:{account_id}:pending" + + +def _get_account_concurrency_limit(account_id: str) -> int: + try: + Account = apps.get_model('api', 'Account') + account = Account.objects.get(id=account_id) + return int((account.usage or {}).get('concurrency', 2)) + except Exception: + return 2 + + +@contextmanager +def account_concurrency_slot(self_task, *, account_id: str, ttl_seconds: int = 21600): + limit = _get_account_concurrency_limit(account_id) + if limit <= 0: + yield False, 0 + return + + token = str(getattr(self_task.request, 'id', None) or secrets.token_hex(16)) + now_ms = int(time.time() * 1000) + ttl_ms = int(ttl_seconds * 1000) + pending_max_age_ms = int(6 * 60 * 60 * 1000) + held_key = _account_semaphore_key(account_id) + pending_key = _account_pending_key(account_id) + result = _ACCT_SEMAPHORE_SCRIPT( + keys=[held_key, pending_key], + args=[token, now_ms, ttl_ms, limit, pending_max_age_ms], + ) + acquired = bool(result and int(result[0]) == 1) + rank = int(result[1]) if result and len(result) > 1 else 0 + try: + yield acquired, rank + finally: + if acquired: + try: + redis_client.zrem(held_key, token) + except Exception: + pass + + +def _reschedule_due_to_concurrency(self_task, *, rank: int) -> None: + base = 2.0 + per_position = 4.0 + max_wait = 240.0 + jitter = random.uniform(0.5, 2.5) + countdown = min(max_wait, base + (max(0, int(rank)) * per_position) + jitter) + + delivery_info = getattr(self_task.request, 'delivery_info', {}) or {} + queue = delivery_info.get('routing_key') or getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED) + kwargs = getattr(self_task.request, 'kwargs', {}) or {} + + self_task.apply_async( + kwargs=kwargs, + countdown=countdown, + queue=queue, + routing_key=queue, + task_id=str(getattr(self_task.request, 'id', '') or secrets.token_hex(16)), + ) + + +def _get_account_id_from_scan_id(scan_id: str) -> str | None: + try: + Scan = apps.get_model('api', 'Scan') + scan = Scan.objects.select_related('page', 'page__account').get(id=scan_id) + return str(scan.page.account.id) if scan.page and scan.page.account else None + except Exception: + return None + + +def _get_account_id_from_test_id(test_id: str) -> str | None: + try: + Test = apps.get_model('api', 'Test') + test = Test.objects.select_related('page', 'page__account').get(id=test_id) + return str(test.page.account.id) if test.page and test.page.account else None + except Exception: + return None + + +def _get_account_id_from_caserun_id(caserun_id: str) -> str | None: + try: + CaseRun = apps.get_model('api', 'CaseRun') + caserun = CaseRun.objects.select_related('account').get(id=caserun_id) + return str(caserun.account.id) if caserun.account else None + except Exception: + return None + + +def _get_account_id_from_page_id(page_id: str) -> str | None: + try: + Page = apps.get_model('api', 'Page') + page = Page.objects.select_related('account').get(id=page_id) + return str(page.account.id) if page.account else None + except Exception: + return None + + +def _get_account_id_from_site_id(site_id: str) -> str | None: + try: + Site = apps.get_model('api', 'Site') + site = Site.objects.select_related('account').get(id=site_id) + return str(site.account.id) if site.account else None + except Exception: + return None + + + + +# setting locking manager to prevent duplicate tasks +@contextmanager +def task_lock(lock_name, timeout=300): + lock = redis_client.lock(lock_name, timeout=timeout) + acquired = lock.acquire(blocking=False) + logger.info(f"Lock {'acquired' if acquired else 'not acquired'} for {lock_name}") + try: + yield acquired + finally: + if acquired: + lock.release() + logger.info(f"Lock released for {lock_name}") + +@contextmanager +def _always_acquired(): + yield True, 0 diff --git a/app/api/signals.py b/app/api/signals.py new file mode 100644 index 00000000..445b9a88 --- /dev/null +++ b/app/api/signals.py @@ -0,0 +1,104 @@ +import threading +from django.db.models.signals import post_save +from django.db import transaction +from django.dispatch import receiver +from .utils.flowr import Flowr +from .utils.agent import Agent +from .tasks import case_pre_run_bg +from .models import * +from cursion import settings + + + + + + +@receiver(post_save, sender=FlowRun) +def flowrun_created(sender, instance, created, **kwargs): + + # defing instance as new flowrun + flowrun = instance + + # check location + if settings.LOCATION == 'us': + + # init Flowr & execute run_next() + if created: + Flowr(flowrun_id=str(flowrun.id)).run_next() + + # return None + return None + + + + +@receiver(post_save, sender=Case) +def case_created(sender, instance, created, **kwargs): + + # defing instance as new case + case = instance + + # check location and created + if settings.LOCATION == 'us' and created: + + # check if Case has processed + if not case.processed: + + # return early if no site association + if not case.site: + case.processed = True + case.save() + return None + + # create process obj + process = Process.objects.create( + site=case.site, + type='case.pre_run', + object_id=str(case.id), + account=case.account, + progress=1 + ) + + # start pre_run for new Case + case_pre_run_bg.delay( + case_id=str(case.id), + process_id=str(process.id) + ) + + # return None + return None + + + + +def _run_agent_response(chat_id: str) -> None: + try: + Agent(chat_id=chat_id).respond() + except Exception as e: + print(f'Error in Agent response: {e}') + + +@receiver(post_save, sender=Chat) +def chat_updated(sender, instance, created, **kwargs): + + # defing instance as new chat + chat = instance + + # check if latest message is sent by user + if len(chat.messages) > 0: + if chat.messages[-1].get('author') == 'user': + + # trigger agent asynchronously after transaction commit + transaction.on_commit( + lambda: threading.Thread( + target=_run_agent_response, + args=(str(chat.id),), + daemon=True + ).start() + ) + + # return None + return None + + + diff --git a/app/api/tasks.py b/app/api/tasks.py index d242b559..e535036a 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,143 +1,3248 @@ -from __future__ import absolute_import, unicode_literals -from typing import Any from celery.utils.log import get_task_logger from celery import shared_task -from celery import Task as BaseTask -from .v1.ops.tasks import ( - create_site_task, create_scan_task, run_html_and_logs_task, - run_vrt_task, run_lighthouse_task, run_yellowlab_task, - create_test_task, create_report_task, delete_report_s3, - delete_site_s3, create_testcase_task, migrate_site_task, - delete_testcase_s3, +from cursion import celery +from .utils.crawler import Crawler +from .utils.scanner import Scanner as S +from .utils.tester import Tester as T +from .utils.reporter import Reporter as R +from .utils.wordpress import Wordpress as W +from .utils.alerter import Alerter +from .utils.caser import Caser +from .utils.autocaser import AutoCaser +from .utils.issuer import Issuer +from .utils.exporter import create_and_send_report_export +from .utils.scanner import ( + _html_and_logs, _vrt, _lighthouse, + _yellowlab +) +from .utils.alerts import * +from .utils.updater import update_flowrun +from .utils.meter import meter_account +from .utils.manager import record_task +from .queue import ( + BaseTaskWithRetry, redis_client, + CELERY_QUEUE_SCHEDULED, CELERY_QUEUE_ON_DEMAND, + account_concurrency_slot, _always_acquired, + _reschedule_due_to_concurrency, task_lock, + _get_account_id_from_scan_id, _get_account_id_from_test_id, + _get_account_id_from_caserun_id, get_task_queue, + apply_async_in_queue, + _get_account_id_from_site_id +) +from .models import * +from functools import reduce +from django.db.models import Q +from django.contrib.auth.models import User +from django.utils import timezone +from datetime import datetime, timedelta, timezone as tz +from kombu.utils.encoding import bytes_to_str +from cursion import settings +import boto3, time, requests, operator, \ +json, stripe, inspect, random, secrets + + + + + + +# setting logger +logger = get_task_logger(__name__) + + +def _flow_obj( + parent: str=None, + obj_id: str=None, + status: str='working', + track_id: str=None, + source_id: str=None + ) -> dict: + """ + Build a normalized FlowRun object payload. + """ + + _id = str(obj_id) if obj_id is not None else None + _source_id = str(source_id) if source_id is not None else _id + _track_id = str(track_id) if track_id is not None else (_id if _id is not None else secrets.token_hex(16)) + _parent = str(parent) if parent is not None else None + + return { + 'parent': _parent, + 'id': _id, + 'source_id': _source_id, + 'track_id': _track_id, + 'status': status + } + + +# setting s3 instance +def s3(): + s3 = boto3.resource('s3', + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + return s3 + + + + +def check_and_increment_resource(account_id: str, resource: str) -> bool: + """ + Adds 1 to the Account.usage.{resource} if + {resource}_allowed has not been reached or + if account.type is 'cloud'. + + Args: + 'account_id' : , + 'resource' : 'scan', 'test', 'caserun', etc + + Returns: + bool, True if resource was incremented. + """ + + # get account + account = Account.objects.get(id=account_id) + + # define defaults + success = False + charge_list = ['caseruns', 'scans', 'tests'] + cloud_types = ['cloud', 'team', 'business'] + + # handle non-paid, cloud accounts + if account.type not in cloud_types: + + # check allowance + if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']): + + # increment and update success + account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}']) + account.save() + success = True + + # handle paid, cloud accounts + if account.type in cloud_types: + + # increment chargable resources + if resource in charge_list: + + # check chargablility + if (int(account.usage[f'{resource}'])) >= int(account.usage[f'{resource}_allowed']): + + # meter resource + meter_account(account.id, 1) + + # increment and update success + account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}']) + account.save() + success = True + + + # increment non-chargable resources + if resource not in charge_list: + + # check allowance + if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']): + + # increment and update success + account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}']) + account.save() + success = True + + # return response + return success + + + + +def check_location(location: str) -> bool: + """ + Determines if task should be executed based on + passed location and current system location (settings.LOCATION). + + Args: + 'location': str + + Returns: + bool (True if task should run) + """ + + # compare location to system + if location == settings.LOCATION: + return True + if location != settings.LOCATION: + return False + + + + +def update_schedule(task_id: str=None) -> None: + """ + Helper function to update Schedule.time_last_run + + Args: + task_id: str + + Returns: + None + """ + if task_id: + try: + last_run = datetime.now(tz.utc) + Schedule.objects.filter(periodic_task_id=task_id).update( + time_last_run=last_run + ) + except Exception as e: + logger.info(e) + return None + + + + +def add_scan_system_data(scan: object=None, kwargs: dict={}) -> dict: + """ + Helper function to build system for passed `Scan`. + + Args: + 'scan': obj + + Returns: + `Scan` + """ + + # build system data + system = { + "tasks": [ + { + "kwargs": kwargs, + "task_id": f"lock:html_and_logs_bg_{scan.id}" if t == 'html' else f"lock:{t}_bg_{scan.id}", + "attempts": 0, + "component": t, + "task_method": "run_html_and_logs_bg" if t == 'html' else f"run_{t}_bg" + } + for t in scan.type if t != 'logs' + ] + } + + # save to scan + scan.system = system + scan.save() + return scan + + + + +def call_local_task_by_name( + task_name : str=None, + kwargs : dict={}, + task_id : str=None + ) -> None: + """ + Helper method to dynamically re-execute local tasks. + + Args: + 'task_name' : str, + 'kwargs' : dict, + 'task_id' : str + + Returns: + `task_name.apply_async(...)` + """ + + task_func = globals()[task_name] + print(f'calling -> task_func.apply_async(kwargs={kwargs}, task_id={task_id})') + queue = get_task_queue(kwargs=kwargs) + return apply_async_in_queue(task_func, kwargs=kwargs, queue=queue, task_id=task_id) + + + + +@shared_task() +def redeliver_failed_tasks() -> None: + """ + Check each un-completed resource (Scans & Tests) + for any celery tasks which are no longer executing & + associated resource.component is null. Once found, + re-run those specific tasks with saved kwargs. If + resource appears complete but is not marked as such, + update `.time_completed` with `timezone.now()` + + Args: + None + + Returns: + None + """ + + # get uncompleted Scans & Tests + scans = Scan.objects.filter(time_completed=None) + tests = Test.objects.filter(time_completed=None).exclude(post_scan__time_completed=None) + flowruns = FlowRun.objects.filter(time_completed=None) + types = ['html_and_logs_bg', 'lighthouse_bg', 'yellowlab_bg', 'vrt_bg'] + + # inspect Celery workers + i = celery.app.control.inspect() + + # fetch active, reserved, scheduled (ETA), & queues tasks + reserved = i.reserved() or {} + active = i.active() or {} + scheduled = i.scheduled() or {} + broker_queues = list( + { + 'celery', + CELERY_QUEUE_SCHEDULED, + CELERY_QUEUE_ON_DEMAND, + getattr(settings, 'CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED), + } + ) + queued = [] + for q in broker_queues: + try: + queued += redis_client.lrange(str(q), 0, -1) + except Exception: + continue + all_tasks = [] + + # gather task IDs from reserved queue + for replica, tasks in reserved.items(): + for task in tasks: + all_tasks.append(task['id']) + + # gather task IDs from active tasks + for replica, tasks in active.items(): + for task in tasks: + all_tasks.append(task['id']) + + # gather task IDs from scheduled/ETA tasks + for replica, tasks in scheduled.items(): + for task in tasks: + req = task.get('request') or {} + if req.get('id'): + all_tasks.append(req['id']) + + # gather redis_ids from queued tasks + for item in queued: + try: + decoded = json.loads(bytes_to_str(item)) + all_tasks.append(decoded['headers']['id']) + except Exception as e: + print(f"Failed to decode task: {e}") + + # iterate through each scan and re-run + # any failed, non-pending jobs + for scan in scans: + + # check for localization + if scan.configs.get('location', 'us') != settings.LOCATION: + continue + + # check each task in system['tasks'] + retried_tasks = 0 + pending_tasks = 0 + test_id = None + alert_id = None + flowrun_id = None + node_index = None + track_id = None + queue = None + components = [] + for task in (scan.system or {}).get('tasks', []): + + # get task_id + task_id = task.get('task_id') + + # get scan.{component} data + if task['component'] == 'yellowlab': + component = scan.yellowlab.get('audits', None) + if task['component'] == 'lighthouse': + component = scan.lighthouse.get('audits', None) + if task['component'] == 'vrt': + component = scan.images + if task['component'] == 'html': + component = scan.html + + # record components + components.append(task['component']) + + # try to get args + test_id = task['kwargs'].get('test_id') or test_id + alert_id = task['kwargs'].get('alert_id') or alert_id + flowrun_id = task['kwargs'].get('flowrun_id') or flowrun_id + node_index = task['kwargs'].get('node_index') or node_index + track_id = task['kwargs'].get('track_id') or track_id + queue = get_task_queue(kwargs=task.get('kwargs') or {}) or queue + + # re-run task if task is not "running", "pending", + # or reached "max_attempts". + if component is None: + + # check if task is "pending" (has task_id in queue) + if task_id in all_tasks: + pending_tasks += 1 + + # check for max attempts + elif task.get('attempts', 0) < settings.MAX_ATTEMPTS: + task_id = f"lock:{task['task_method'].replace('run_', '')}_{scan.id}" + call_local_task_by_name(task["task_method"], task["kwargs"], task_id) + retried_tasks += 1 + + # try to get test_id + if not test_id and Test.objects.filter(post_scan=scan, time_completed=None).exists(): + test_id = Test.objects.filter(post_scan=scan, time_completed=None)[0].id + + # mark scan complete if checks pass + if retried_tasks == 0 and pending_tasks == 0: + logger.info(f'marking scan as complete') + scan.time_completed = timezone.now() + scan.save() + + # update site and page with most recent data + update_site_and_page_info( + resource='scan', + page_id=str(scan.page.id) + ) + + # execute `run_test()` if test_id present + if test_id: + logger.info(f'executing run_test() from `post_scan` in `retry_tasks`') + queue = queue or get_task_queue(kwargs={'_queue': None}) + + # preserve object identity for FlowRun merging + if track_id is None: + try: + _test = Test.objects.get(id=test_id) + _task_kwargs = (((_test.system or {}).get('tasks') or [{}])[0].get('kwargs') or {}) + track_id = _task_kwargs.get('track_id') or str(test_id) + except Exception: + track_id = str(test_id) + + apply_async_in_queue( + run_test, + kwargs={ + 'test_id': str(test_id), + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'track_id': track_id, + '_queue': queue, + }, + queue=queue, + task_id=f'lock:run_test_{test_id}', + ) + + # iterate through each test and re-run if failed + for test in tests: + + # check for localization + if test.post_scan.configs.get('location', 'us') != settings.LOCATION: + continue + + # check each task in system['tasks'] + retried_tasks = 0 + pending_tasks = 0 + for task in (test.system or {}).get('tasks', []): + + # define task_id + task_id = task.get('task_id') + + # check if task is "pending" (has task_id in queue) + if task_id in all_tasks: + pending_tasks += 1 + + # check for post_scan related task_ids in all_tasks + elif any(f'lock:{t}_{test.post_scan.id}' in all_tasks for t in types): + pending_tasks += 1 + + # check for max attempts + elif task.get('attempts', 0) < settings.MAX_ATTEMPTS: + task_id = f'lock:run_test_{test.id}' + call_local_task_by_name(task["task_method"], task["kwargs"], task_id) + retried_tasks += 1 + + # mark test incomplete if checks pass + if retried_tasks == 0 and pending_tasks == 0: + test.time_completed = timezone.now() + test.status = 'incomplete' + test.save() + + # update site and page with most recent data + update_site_and_page_info( + resource='test', + page_id=str(test.page.id) + ) + + # get flowrun info + first_task = ((test.system or {}).get('tasks') or [{}])[0] + kwargs = first_task.get('kwargs') or {} + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + track_id = kwargs.get('track_id') or str(test.id) + + # update FlowRun if present + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': ( + f'test for {test.page.page_url} completed with status: '+ + f'⏺️ INCOMPLETE | test_id: {str(test.id)}' + ), + 'objects': [_flow_obj( + parent=str(test.page.id), + obj_id=str(test.id), + source_id=str(test.id), + track_id=track_id, + status='incomplete' + )] + }) + + # iterate through each FlowRun + for flowrun in flowruns: + + # get last recorded log + print(f'flowrun incomplete ID: {flowrun.id}') + + # check the objects of the current working node and confirm their status. + + + + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None: + """ + Takes a newly created `Site`, initiates a Crawl and + initial `Scan` for each crawled page + + Args: + site_id: str, + configs: dict + + Returns: None + """ + + # getting site and updating for time_crawl_start + site = Site.objects.get(id=site_id) + site.time_crawl_started = datetime.now(tz.utc) + site.time_crawl_completed = None + site.save() + + # get max_urls + max_urls = site.account.usage['pages_allowed'] + + # crawl site + pages = Crawler(url=site.site_url, max_urls=max_urls).get_links() + + queue = get_task_queue(self.request) + + # create pages and scans + for url in pages: + + # add new page + if not Page.objects.filter(site=site, page_url=url).exists(): + page = Page.objects.create( + site=site, + page_url=url, + user=site.user, + account=site.account, + ) + + # check resouce allowance + if check_and_increment_resource(site.account.id, 'scans'): + + # create initial scan + scan = Scan.objects.create( + site=site, + page=page, + type=settings.TYPES, + configs=configs + ) + + apply_async_in_queue( + scan_page_bg, + kwargs={'scan_id': str(scan.id), '_queue': queue}, + queue=queue, + ) + + # update page info + page.info["latest_scan"]["id"] = str(scan.id) + page.info["latest_scan"]["time_created"] = str(scan.time_created) + page.save() + + # updating site status + site.time_crawl_completed = timezone.now() + site.save() + + logger.info('Added site and all pages') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None: + """ + Takes an existing `Site`, initiates a new Crawl and + initial `Scan` for each newly added page + + Args: + site_id: str, + configs: dict + + Returns: None + """ + + # getting site and updating for time_crawl_start + site = Site.objects.get(id=site_id) + site.time_crawl_started = timezone.now() + site.time_crawl_completed = None + site.save() + + # get pages_allowed + pages_allowed = site.account.usage['pages_allowed'] + + # getting old pages for comparison + old_pages = Page.objects.filter(site=site) + old_urls = [] + for p in old_pages: + old_urls.append(p.page_url) + + # crawl site + new_urls = Crawler(url=site.site_url, max_urls=pages_allowed).get_links() + add_urls = [] + + queue = get_task_queue(self.request) + + # checking for duplicates + for url in new_urls: + if not url in old_urls: + add_urls.append(url) + + # loop thorugh crawled pages + # and add if not present + current_count = len(old_urls) + for url in add_urls: + + # add new page if room exists + if current_count < pages_allowed: + page = Page.objects.create( + site=site, + page_url=url, + user=site.user, + tags=[], + account=site.account, + ) + + # check resouce allowance + if check_and_increment_resource(site.account.id, 'scans'): + + # create initial scan + scan = Scan.objects.create( + site=site, + page=page, + type=settings.TYPES, + configs=configs + ) + + apply_async_in_queue( + scan_page_bg, + kwargs={'scan_id': str(scan.id), '_queue': queue}, + queue=queue, + ) + page.info["latest_scan"]["id"] = str(scan.id) + page.info["latest_scan"]["time_created"] = str(scan.time_created) + page.save() + + # increment + current_count += 1 + + # updating site status + site.time_crawl_completed = timezone.now() + site.save() + + logger.info('crawled site and added pages') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def update_site_and_page_info( + self, + resource: str='all', + site_id: str=None, + page_id: str=None, + ) -> None: + """ + Updates the site and or page `latest_scan` & `latest_test` info + depending on scope. + + Args: + "resource" : str (OPTIONAL), + "site_id" : str (OPTIONAL), + "page_id" : str (OPTIONAL) + + Returns: None + """ + + # defaults + site = None + page = None + pages = [] + scans = [] + tests = [] + + # get associated site + if site_id: + site = Site.objects.get(id=site_id) + pages = Page.objects.filter(site=site) + + # get associated page + if page_id: + page = Page.objects.get(id=page_id) + site = page.site + pages = Page.objects.filter(site=site) + + # get latest tests & scans of pages + for p in pages: + + # set defaults + latest_scan = None + latest_test = None + site_avg_scan_score = None + + if Test.objects.filter(page=p).exists() and \ + (resource == 'test' or resource == 'all'): + _test = Test.objects.filter(page=p).exclude( + time_completed=None + ).order_by('-time_completed') + if len(_test) > 0: + if _test[0].status: + # update latest_test + latest_test = _test[0] + if _test[0].score: + # add to tests[] + tests.append(_test[0].score) + + if Scan.objects.filter(page=p).exists() and \ + (resource == 'scan' or resource == 'all'): + _scan = Scan.objects.filter(page=p).exclude( + time_completed=None + ).order_by('-time_completed') + if len(_scan) > 0: + if _scan[0].score: + # add to scans[] + scans.append(_scan[0].score) + # update latest_scan + latest_scan = _scan[0] + + # update single page if passed + if page: + + # checking if current p is page + if page.id == p.id: + + # latest_scan info + if latest_scan: + page.info['latest_scan']['id'] = str(latest_scan.id) + page.info['latest_scan']['time_created'] = str(latest_scan.time_created) + page.info['latest_scan']['time_completed'] = str(latest_scan.time_completed) + page.info['latest_scan']['score'] = latest_scan.score + logger.info(f'updating {page.page_url} with scan.score -> {latest_scan.score}') + if latest_scan is None and (resource == 'scan' or resource == 'all'): + page.info['latest_scan']['id'] = None + page.info['latest_scan']['time_created'] = None + page.info['latest_scan']['time_completed'] = None + page.info['latest_scan']['score'] = None + logger.info(f'updating {page.page_url} with scan.score -> {None}') + + # latest_test info + if latest_test: + page.info['latest_test']['id'] = str(latest_test.id) + page.info['latest_test']['time_created'] = str(latest_test.time_created) + page.info['latest_test']['time_completed'] = str(latest_test.time_completed) + page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100) if latest_test.score else None + page.info['latest_test']['status'] = latest_test.status + logger.info(f'updating {p.page_url} with test.score -> {latest_test.score}') + if latest_test is None and (resource == 'test' or resource == 'all'): + page.info['latest_test']['id'] = None + page.info['latest_test']['time_created'] = None + page.info['latest_test']['time_completed'] = None + page.info['latest_test']['score'] = None + page.info['latest_test']['status'] = None + logger.info(f'updating {p.page_url} with test.score -> {None}') + + # save page + page.save() + + # update site with new scan info + if len(scans) > 0: + # calc site average of latest_scan.score + site_avg_scan_score = round((sum(scans)/len(scans)) * 100) / 100 + logger.info(f'updating site with new scan score -> {site_avg_scan_score}') + + # latest_scan info + if latest_scan: + site.info['latest_scan']['id'] = str(latest_scan.id) + site.info['latest_scan']['time_created'] = str(latest_scan.time_created) + site.info['latest_scan']['time_completed'] = str(latest_scan.time_completed) + site.info['latest_scan']['score'] = site_avg_scan_score + if latest_scan is None and (resource == 'scan' or resource == 'all'): + site.info['latest_scan']['id'] = None + site.info['latest_scan']['time_created'] = None + site.info['latest_scan']['time_completed'] = None + site.info['latest_scan']['score'] = None + + # update site info + if latest_test: + site.info['latest_test']['id'] = str(latest_test.id) + site.info['latest_test']['time_created'] = str(latest_test.time_created) + site.info['latest_test']['time_completed'] = str(latest_test.time_completed) + site.info['latest_test']['score'] = latest_test.score + site.info['latest_test']['status'] = latest_test.status + if latest_test is None and (resource == 'test' or resource == 'all'): + site.info['latest_test']['id'] = None + site.info['latest_test']['time_created'] = None + site.info['latest_test']['time_completed'] = None + site.info['latest_test']['score'] = None + site.info['latest_test']['status'] = None + + # save info + site.save() + + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def update_scan_score(self, scan_id: str) -> None: + """ + Method to calculate the average health score and update + for the passed scan_id + + Args: + 'scan_id': str + + Returns: None + """ + + # setting defaults + score = None + scores = [] + scan = Scan.objects.get(id=scan_id) + + # get latest scan scores + if scan.lighthouse['scores']['average'] is not None: + scores.append(scan.lighthouse['scores']['average']) + if scan.yellowlab['scores']['globalScore'] is not None: + scores.append(scan.yellowlab['scores']['globalScore']) + + # calc average score + if len(scores) > 0: + score = sum(scores)/len(scores) + + # save to scan + scan.score = score + scan.save() + + # returning scan + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def scan_page_bg( + self, + scan_id : str=None, + test_id : str=None, + alert_id : str=None, + flowrun_id : str=None, + node_index : str=None, + _queue : str=None, + ) -> None: + """ + Runs all the requested `Scan` components + of the passed `Scan`. + + Args: + scan_id : str, + test_id : str, + alert_id : str, + configs : dict, + flowrun_id : str, + node_index : str + + Returns: None + """ + + # get scan object + scan = Scan.objects.get(id=scan_id) + + queue = get_task_queue(self.request, kwargs={'_queue': _queue} if _queue else None) + + # run each scan component in parallel + if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type: + apply_async_in_queue( + run_html_and_logs_bg, + kwargs={ + 'scan_id' : scan_id, + 'test_id' : test_id, + 'alert_id' : alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue' : queue, + }, + queue=queue, + task_id=f'lock:html_and_logs_bg_{scan_id}', + ) + if 'lighthouse' in scan.type or 'full' in scan.type: + apply_async_in_queue( + run_lighthouse_bg, + kwargs={ + 'scan_id' : scan_id, + 'test_id' : test_id, + 'alert_id' : alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue' : queue, + }, + queue=queue, + task_id=f'lock:lighthouse_bg_{scan_id}', + ) + if 'yellowlab' in scan.type or 'full' in scan.type: + apply_async_in_queue( + run_yellowlab_bg, + kwargs={ + 'scan_id' : scan_id, + 'test_id' : test_id, + 'alert_id' : alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue' : queue, + }, + queue=queue, + task_id=f'lock:yellowlab_bg_{scan_id}', + ) + if 'vrt' in scan.type or 'full' in scan.type: + apply_async_in_queue( + run_vrt_bg, + kwargs={ + 'scan_id' : scan_id, + 'test_id' : test_id, + 'alert_id' : alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue' : queue, + }, + queue=queue, + task_id=f'lock:vrt_bg_{scan_id}', + ) + + logger.info('started scan component tasks') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_scan( + self, + scan_id: str=None, + page_id: str=None, + type: list=settings.TYPES, + alert_id: str=None, + configs: str=None, + tags: str=None, + ) -> None: + """ + Runs a `Scan` using Scanner.build_scan() + where each component is run in sequence. + + Args: + scan_id : str, + page_id : str, + type : list, + alert_id : str, + configs : dict, + tags : list, + + Returns: None + """ + + # get scan if scan_id present + if scan_id is not None: + created_scan = Scan.objects.get(id=scan_id) + + # create scan if page_id present + elif page_id is not None: + page = Page.objects.get(id=page_id) + created_scan = Scan.objects.create( + site=page.site, + page=page, + type=type, + configs=configs, + tags=tags, + ) + + # run scan and alert if necessary + scan = S(scan=created_scan).build_scan() + if alert_id and alert_id != 'None': + logger.info('running alert from `task.create_scan`') + Alerter(alert_id=alert_id, object_id=scan.id).run_alert() + + logger.info('Created new scan of site') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_scan_bg(self, **kwargs) -> None: + """ + Creates 1 or more `Scans` depending on + the scope (page, site or account). Used with `Schedules` + + Args: + 'scope' : str + 'resources' : list + 'tags' : list + 'account_id' : strx + 'type' : list, + 'configs' : dict, + 'tags' : list, + 'alert_id' : str, + 'task_id' : str, + 'flowrun_id' : str, + 'node_index : str + + Returns: None + """ + + # get data from kwargs + scope = kwargs.get('scope') + resources = kwargs.get('resources', []) + tags = kwargs.get('tags', []) + account_id = kwargs.get('account_id') + type = kwargs.get('type') + configs = kwargs.get('configs') + alert_id = kwargs.get('alert_id') + task_id = kwargs.get('task_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + # check for redis lock + redis_id = task_id if task_id else secrets.token_hex(8) + lock_name = f"lock:create_scan_bg_{redis_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # checking location + if not check_location(configs.get('location', settings.LOCATION)): + logger.info('Not running due to location param') + return None + + queue = get_task_queue(self.request, kwargs) + + # setting defaults + pages = [] + sites = [] + objects = [] + + # get account if account_id exists + if account_id: + account = Account.objects.get(id=account_id) + + # iterating through resources + # and adding to sites or pages + if len(resources) > 0: + for item in resources: + + # adding to pages + if item['type'] == 'page': + try: + pages.append( + Page.objects.get(id=item['id']) + ) + except Exception as e: + logger.warning(e) + + # adding to sites + if item['type'] == 'site': + try: + sites.append( + Site.objects.get(id=item['id']) + ) + except Exception as e: + logger.warning(e) + + # iterating through tags + # and adding to sites or pages + if len(tags) > 0: + for tag in tags: + + # adding to pages + try: + pages += Page.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + + # adding to sites + try: + sites += Site.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + + # grabbing all sites because no + # resources/tags were specified and scope is "account" + if len(resources) == 0 and len(tags) == 0 and scope == 'account': + sites = Site.objects.filter(account=account) + + # get all pages from existing sites + for site in sites: + pages += Page.objects.filter(site=site).exclude(id__in=[str(p.id) for p in pages]) + + # creating scans for each page + for page in pages: + + # check resource + if check_and_increment_resource(page.account.id, 'scans'): + + # create Scan obj + scan = Scan.objects.create( + site=page.site, + page=page, + type=type, + tags=tags, + configs=configs + ) + + # update scan with system data + add_scan_system_data( + scan=scan, + kwargs={ + 'scan_id': str(scan.id), + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue': queue, + } + ) + + # updating latest_scan info for page + page.info['latest_scan']['id'] = str(scan.id) + page.info['latest_scan']['time_created'] = str(timezone.now()) + page.info['latest_scan']['time_completed'] = None + page.info['latest_scan']['score'] = None + page.info['latest_scan']['score'] = None + page.save() + + # updating latest_scan info for site + page.site.info['latest_scan']['id'] = str(scan.id) + page.site.info['latest_scan']['time_created'] = str(timezone.now()) + page.site.info['latest_scan']['time_completed'] = None + page.site.save() + + # adding objects + objects.append(_flow_obj( + parent=str(scan.page.id), + obj_id=str(scan.id), + source_id=str(scan.id), + track_id=str(scan.id), + status='working' + )) + + # init scan page in background + apply_async_in_queue( + scan_page_bg, + kwargs={ + 'scan_id': str(scan.id), + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue': queue, + }, + queue=queue, + ) + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'objects': objects, + 'node_status': 'working' if len(objects) > 0 else 'failed', + 'message': f'starting {len(objects)} scans for {page.site.site_url} | run_id: {flowrun_id}' + }) + + # update schedule if task_id is not None + update_schedule(task_id=task_id) + + logger.info('created new Scans') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_html_and_logs_bg( + self, + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None, + **kwargs + ) -> None: + """ + Runs the html & logs components of the passed `Scan` + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str, + **kwargs + + Returns: None + """ + + # get kwargs data if no scan_id + if scan_id is None: + scan_id = kwargs.get('scan_id') + test_id = kwargs.get('test_id') + alert_id = kwargs.get('alert_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # check redis task lock + lock_name = f"lock:html_and_logs_bg_{scan_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # save & check sys data + max_reached = record_task( + resource_type='scan', + resource_id=str(scan_id), + task_id=str(self.request.id), + task_method=str(inspect.stack()[0][3]), + kwargs={ + 'scan_id': str(scan_id) if scan_id is not None else None, + 'test_id': str(test_id) if test_id is not None else None, + 'alert_id': str(alert_id) if alert_id is not None else None, + 'flowrun_id': str(flowrun_id) if flowrun_id is not None else None, + 'node_index': str(node_index) if node_index is not None else None + } + ) + + # return early if max_attempts reached + if max_reached: + logger.info('max attempts reach for html & logs component') + return None + + # run html and logs component + _html_and_logs(scan_id, test_id, alert_id, flowrun_id, node_index) + + logger.info('ran html & logs component') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_vrt_bg( + self, + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None, + **kwargs + ) -> None: + """ + Runs the VRT component of the passed `Scan` + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str, + **kwargs + + Returns: None + """ + + # get kwargs data if no scan_id + if scan_id is None: + scan_id = kwargs.get('scan_id') + test_id = kwargs.get('test_id') + alert_id = kwargs.get('alert_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # check redis task lock + lock_name = f"lock:vrt_bg_{scan_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # save sys data + max_reached = record_task( + resource_type='scan', + resource_id=str(scan_id), + task_id=str(self.request.id), + task_method=str(inspect.stack()[0][3]), + kwargs={ + 'scan_id': str(scan_id) if scan_id is not None else None, + 'test_id': str(test_id) if test_id is not None else None, + 'alert_id': str(alert_id) if alert_id is not None else None, + 'flowrun_id': str(flowrun_id) if flowrun_id is not None else None, + 'node_index': str(node_index) if node_index is not None else None + } + ) + + # return early if max_attempts reached + if max_reached: + logger.info('max attempts reach for vrt component') + return None + + # run VRT component + _vrt(scan_id, test_id, alert_id, flowrun_id, node_index) + + logger.info('ran vrt component') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_lighthouse_bg( + self, + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None, + **kwargs + ) -> None: + """ + Runs the lighthouse component of the passed `Scan` + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str, + **kwargs + + Returns: None + """ + + # get kwargs data if no scan_id + if scan_id is None: + scan_id = kwargs.get('scan_id') + test_id = kwargs.get('test_id') + alert_id = kwargs.get('alert_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # check redis task lock + lock_name = f"lock:lighthouse_bg_{scan_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # save sys data + max_reached = record_task( + resource_type='scan', + resource_id=str(scan_id), + task_id=str(self.request.id), + task_method=str(inspect.stack()[0][3]), + kwargs={ + 'scan_id': str(scan_id) if scan_id is not None else None, + 'test_id': str(test_id) if test_id is not None else None, + 'alert_id': str(alert_id) if alert_id is not None else None, + 'flowrun_id': str(flowrun_id) if flowrun_id is not None else None, + 'node_index': str(node_index) if node_index is not None else None + } + ) + + # return early if max_attempts reached + if max_reached: + logger.info('max attempts reach for lighthouse component') + return None + + # run lighthouse component + _lighthouse(scan_id, test_id, alert_id, flowrun_id, node_index) + + logger.info('ran lighthouse component') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_yellowlab_bg( + self, + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None, + **kwargs + ) -> None: + """ + Runs the yellowlab component of the passed `Scan` + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str, + **kwargs + + Returns: None + """ + + # get kwargs data if no scan_id + if scan_id is None: + scan_id = kwargs.get('scan_id') + test_id = kwargs.get('test_id') + alert_id = kwargs.get('alert_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + account_id = _get_account_id_from_scan_id(str(scan_id)) if scan_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # check redis task lock + lock_name = f"lock:yellowlab_bg_{scan_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # save sys data + max_reached = record_task( + resource_type='scan', + resource_id=str(scan_id), + task_id=str(self.request.id), + task_method=str(inspect.stack()[0][3]), + kwargs={ + 'scan_id': str(scan_id) if scan_id is not None else None, + 'test_id': str(test_id) if test_id is not None else None, + 'alert_id': str(alert_id) if alert_id is not None else None, + 'flowrun_id': str(flowrun_id) if flowrun_id is not None else None, + 'node_index': str(node_index) if node_index is not None else None + } + ) + + # return early if max_attempts reached + if max_reached: + logger.info('max attempts reach for yellowlab component') + return None + + # run yellowlab component + _yellowlab(scan_id, test_id, alert_id, flowrun_id, node_index) + + logger.info('ran yellowlab component') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_test( + self, + test_id: str, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None , + track_id: str=None, + **kwargs + ) -> None: + """ + Primary executor for running a `Test`. + Compatible with `FlowRuns` + + Args: + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str, + **kwargs + + Returns: None + """ + # get kwargs data if no test_id + if test_id is None: + test_id = kwargs.get('test_id') + alert_id = kwargs.get('alert_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + track_id = kwargs.get('track_id') + + account_id = _get_account_id_from_test_id(str(test_id)) if test_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # check redis task lock + lock_name = f"lock:run_test_{test_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # save sys data + max_reached = record_task( + resource_type='test', + resource_id=str(test_id), + task_id=str(self.request.id), + task_method=str(inspect.stack()[0][3]), + kwargs={ + 'test_id': str(test_id), + 'alert_id': str(alert_id) if alert_id is not None else None, + 'flowrun_id': str(flowrun_id) if flowrun_id is not None else None, + 'node_index': str(node_index) if node_index is not None else None, + 'track_id': str(track_id) if track_id is not None else None + } + ) + + # return early if max_attempts reached + if max_reached: + logger.info('max attempts reach for Tester') + return None + + # get test + test = Test.objects.get(id=test_id) + + # idempotency guard: once a Test is complete, do not execute it again. + if test.time_completed is not None: + logger.info(f'skipping run_test for completed test_id: {str(test_id)}') + return None + + # define objects for flowrun + objects = [_flow_obj( + parent=str(test.page.id), + obj_id=str(test_id), + source_id=str(test_id), + track_id=track_id or str(test_id), + status='working' + )] + + # update flowrun + if flowrun_id and flowrun_id != 'None': + time.sleep(random.uniform(0.1, 5)) + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': f'starting test comparison algorithm for {test.page.page_url} | test_id: {str(test_id)}', + 'objects': objects + }) + + # execute test + logger.info('\n---------------\nStarting Test...\n---------------\n') + test = T(test=test).run_test() + + # update FlowRun if passed + if flowrun_id and flowrun_id != 'None': + objects[-1]['status'] = test.status + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': ( + f'test for {test.page.page_url} completed with status: '+ + f'{"❌ FAILED" if test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}' + ), + 'objects': objects + }) + + # execute Alert if passed + if alert_id and alert_id != 'None': + logger.info('running alert from `task.run_test`') + Alerter(alert_id=alert_id, object_id=str(test.id)).run_alert() + + logger.info('Test completed') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_test( + self, + test_id: str=None, + page_id: str=None, + alert_id: str=None, + configs: dict=settings.CONFIGS, + type: list=settings.TYPES, + pre_scan: str=None, + post_scan: str=None, + tags: list=None, + threshold: float=settings.TEST_THRESHOLD, + flowrun_id: str=None, + node_index: str=None, + track_id: str=None, + _queue: str=None, + ) -> None: + """ + Creates a `post_scan` if necessary, waits for completion, + and runs a `Test` + + Args: + test_id : str, + page_id : str, + alert_id : str, + configs : dict, + type : list, + pre_scan : str, + post_scan : str, + tags : list, + threshold : float, + flowrun_id : str, + node_index : str + + Returns: None + """ + + # setting defaults + created_test = None + objects = [] + queue = get_task_queue(self.request, kwargs={'_queue': _queue} if _queue else None) + + # get or create a Test + if test_id is not None: + created_test = Test.objects.get(id=test_id) + page = created_test.page + elif page_id is not None: + page = Page.objects.get(id=page_id) + created_test = Test.objects.create( + site=page.site, + page=page, + type=type, + tags=tags, + threshold=float(threshold), + status='working' + ) + + # adding objects + objects.append(_flow_obj( + parent=str(page.id), + obj_id=str(created_test.id), + source_id=str(created_test.id), + track_id=track_id or str(created_test.id), + status='working' + )) + + # create system data for new Scan & Test (may not be used) + test_system = { + "tasks": [ + { + "kwargs": { + "test_id": str(created_test.id), + "alert_id": alert_id, + "flowrun_id": flowrun_id, + "node_index": node_index, + "track_id": track_id or str(created_test.id) + }, + "task_id": f"lock:run_test_{created_test.id}", + "attempts": 0, + "component": "test", + "task_method": "run_test" + } + ] + } + + # get pre_ & post_ scans + if pre_scan is not None: + pre_scan = Scan.objects.get(id=pre_scan) + if post_scan is not None: + post_scan = Scan.objects.get(id=post_scan) + if post_scan is None or pre_scan is None: + if pre_scan is None: + + # get latest pre_scan (matching page, type, and window_size) + pre_scan = ( + Scan.objects.filter( + page=page, + configs__window_size=configs.get('window_size') + ) + .filter( + reduce(operator.and_, (Q(type__contains=[t]) for t in type)) + ) + .exclude(time_completed=None) + .order_by('-time_completed') + .first() + ) + + # check for pre_scan existance + if not pre_scan: + + # create new scan if none exists + new_scan = Scan.objects.create( + site=page.site, + page=page, + tags=tags, + type=settings.TYPES, + configs=configs + ) + + # update scan with system data + add_scan_system_data( + scan=new_scan, + kwargs={ + 'scan_id': str(new_scan.id), + 'alert_id': None, + 'flowrun_id': flowrun_id, + 'node_index': node_index + } + ) + + # init Scan process + scan_page_bg( + scan_id=new_scan.id, + flowrun_id=flowrun_id, + node_index=node_index, + _queue=queue, + ) + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'objects': objects, + 'message': ( + f'❌ test for {page.page_url} could not start because there was '+ + f'no pre_scan available - starting new scan instead' + ) + }) + + # remove created_test + created_test.delete() + + # return None + logger.info('no pre_scan available to create Test with') + return None + + + # check and increment resources + if not check_and_increment_resource(page.account.id, 'scans'): + + # update objects + objects[-1]['status'] = 'failed' + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'objects': objects, + 'message': ( + f'❌ test for {page.page_url} could not start because this account has reached '+ + f'max_allowed_scans for this billing cycle' + ) + }) + + # remove created_test + created_test.delete() + + # return None + logger.info('no more scans usage available') + return None + + # create new post_scan + post_scan = Scan.objects.create( + site=page.site, + page=page, + tags=tags, + type=type, + configs=configs + ) + + # update scan with system data + add_scan_system_data( + scan=post_scan, + kwargs={ + 'scan_id': str(post_scan.id), + 'test_id': str(created_test.id), + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index + } + ) + + # run Scan & Test tasks + scan_page_bg( + scan_id=post_scan.id, + test_id=created_test.id, + alert_id=alert_id, + flowrun_id=flowrun_id, + node_index=node_index, + _queue=queue, + ) + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'objects': objects, + 'message': ( + f'test starting for {page.page_url} | '+ + f'run_id: {flowrun_id}' + ) + }) + + # updating parired scans + pre_scan.paired_scan = post_scan + post_scan.paired_scan = pre_scan + pre_scan.save() + post_scan.save() + + # updating test object + created_test.type = type + created_test.pre_scan = pre_scan + created_test.post_scan = post_scan + created_test.system = test_system + created_test.save() + + # check if pre and post scan are complete and start test if True + if pre_scan.time_completed is not None and post_scan.time_completed is not None: + apply_async_in_queue( + run_test, + kwargs={ + 'test_id': str(created_test.id), + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'track_id': track_id or str(created_test.id), + '_queue': queue, + }, + queue=queue, + task_id=f'lock:run_test_{created_test.id}', + ) + + logger.info('Began Scan/Test process') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_test_bg(self, **kwargs) -> None: + """ + Depending on the scope, run create_test() for + all requested pages. + + Args: + scope : str + resources : list + tags : list + account_id : str + test_id : str + type : list + configs : dict + alert_id : str + pre_scan : str + post_scan : str + threshold : float + task_id : str + flowrun_id : str + node_index : str + + Returns: None + """ + + # get data + scope = kwargs.get('scope') + resources = kwargs.get('resources', []) + tags = kwargs.get('tags', []) + account_id = kwargs.get('account_id') + test_id = kwargs.get('test_id') + type = kwargs.get('type') + configs = kwargs.get('configs') + threshold = kwargs.get('threshold') + alert_id = kwargs.get('alert_id') + pre_scan = kwargs.get('pre_scan') + post_scan = kwargs.get('post_scan') + task_id = kwargs.get('task_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + # check for redis lock + redis_id = task_id if task_id else secrets.token_hex(8) + lock_name = f"lock:create_test_bg_{redis_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # checking location + if not check_location(configs.get('location', settings.LOCATION)): + logger.info('Not running due to location param') + return None + + queue = get_task_queue(self.request, kwargs) + + # create test if none was passed + if test_id is None: + + # setting defaults + pages = [] + sites = [] + objects = [] + failed = 0 + + # get account if account_id exists + if account_id: + account = Account.objects.get(id=account_id) + + # iterating through resources + # and adding to sites or pages + if len(resources) > 0: + for item in resources: + + # adding to pages + if item['type'] == 'page': + try: + pages.append( + Page.objects.get(id=item['id']) + ) + except Exception as e: + logger.info(e) + + # adding to sites + if item['type'] == 'site': + try: + sites.append( + Site.objects.get(id=item['id']) + ) + except Exception as e: + logger.info(e) + + # iterating through tags + # and adding to sites or pages + if len(tags) > 0: + for tag in tags: + + # adding to pages + try: + pages += Page.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + + # adding to sites + try: + sites += Site.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + + # grabbing all sites because no + # resources were specified and scope is "account" + if len(resources) == 0 and scope == 'account': + sites = Site.objects.filter(account=account) + + # get all pages from existing sites + for site in sites: + pages += Page.objects.filter(site=site).exclude(id__in=[str(p.id) for p in pages]) + + # create a test for each page + for page in pages: + + flow_track_id = secrets.token_hex(16) + objects.append(_flow_obj( + parent=str(page.id), + obj_id=None, + source_id=str(page.id), + track_id=flow_track_id, + status='working' + )) + + # check resource + if check_and_increment_resource(page.account.id, 'tests'): + + # updating latest_test info for page + page.info['latest_test']['id'] = 'placeholder' + page.info['latest_test']['time_created'] = str(timezone.now()) + page.info['latest_test']['time_completed'] = None + page.info['latest_test']['score'] = None + page.info['latest_test']['status'] = 'working' + page.save() + + # updating latest_test info for site + page.site.info['latest_test']['id'] = 'placeholder' + page.site.info['latest_test']['time_created'] = str(timezone.now()) + page.site.info['latest_test']['time_completed'] = None + page.site.info['latest_test']['score'] = None + page.site.info['latest_test']['status'] = 'working' + page.site.save() + + # create test + create_test( + page_id=str(page.id), + type=type, + configs=configs, + tags=tags, + threshold=float(threshold), + pre_scan=pre_scan, + post_scan=post_scan, + alert_id=str(alert_id), + flowrun_id=str(flowrun_id), + node_index=node_index, + track_id=flow_track_id, + _queue=queue, + ) + + else: + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': ( + f'❌ test for {page.page_url} could not start because this account has reached '+ + f'max_allowed_tests for this billing cycle' + ) + }) + + # update last object + failed += 1 + objects[-1]['status'] = 'failed' + logger.info('maxed tests reached') + update_schedule(task_id=task_id) + return None + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'objects': objects, + 'node_status': 'working', + 'message': f'created {str(len(objects) - failed)} tests for {page.site.site_url} | run_id: {flowrun_id}' + }) + + # get test and run + if test_id: + test = Test.objects.get(id=test_id) + apply_async_in_queue( + create_test, + kwargs={ + 'test_id': str(test_id), + 'page_id': str(test.page.id), + 'type': type, + 'configs': configs, + 'tags': tags, + 'threshold': float(threshold), + 'pre_scan': pre_scan, + 'post_scan': post_scan, + 'alert_id': str(alert_id), + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + '_queue': queue, + }, + queue=queue, + ) + + # update schedule if task_id is not None + update_schedule(task_id=task_id) + + logger.info('Created new Tests') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_report( + self, + site_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None, + track_id: str=None, + _queue: str=None, + **kwargs, + ) -> None: + """ + Generates a new PDF `Report` of the requested `Site` + and runs the associated `Alert` if requested + + Args: + site_id : str, + alert_id : str, + flowrun_id : str + node_index : str + + Returns: None + """ + if not site_id: + logger.warning('create_report skipped: missing site_id') + return None + + account_id = _get_account_id_from_site_id(str(site_id)) if site_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # get site + site = Site.objects.get(id=site_id) + + lookback_days = kwargs.get('lookback_days', 7) + report_type = kwargs.get('type', ['issues', 'tests', 'caseruns', 'performance']) + text_color = kwargs.get('text_color', '#24262d') + background_color = kwargs.get('background_color', '#e1effd') + highlight_color = kwargs.get('highlight_color', '#ffffff') + + # create report obj + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + "lookback_days": lookback_days, + } + report = Report.objects.create( + user=site.user, + site=site, + account=site.account, + info=info, + type=report_type + ) + + # generate report PDF + resp = R(report=report).generate_report() + + # run alert + if alert_id and alert_id != 'None': + Alerter(alert_id, str(report.id)).run_alert() + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': f'report {"created" if resp["success"] else "not created"} for {site.site_url} | report_id: {str(report.id)}', + 'objects': [_flow_obj( + parent=str(site.id), + obj_id=str(report.id), + source_id=str(report.id), + track_id=track_id or str(report.id), + status='passed' if resp['success'] else 'failed' + )] + }) + + logger.info('Created new report of site') + return None + + + + +@shared_task +def create_report_bg(**kwargs) -> None: + """ + Creates new `Reports` for the requested `Sites` + + Args: + 'scope' : str, + 'resources' : list + 'tags' : list + 'account_id' : str + 'alert_id' : str + 'task_id' : str + 'flowrun_id' : str + 'node_index' : str + + Returns: + None + """ + + # get data + scope = kwargs.get('scope') + resources = kwargs.get('resources', []) + tags = kwargs.get('tags', []) + account_id = kwargs.get('account_id') + alert_id = kwargs.get('alert_id') + task_id = kwargs.get('task_id') + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + # check for redis lock + redis_id = task_id if task_id else secrets.token_hex(8) + lock_name = f"lock:create_report_bg_{redis_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + queue = get_task_queue(kwargs=kwargs) + + # setting defaults + sites = [] + objects = [] + report_track_map = {} + + # get account if account_id exists + if account_id: + account = Account.objects.get(id=account_id) + + logger.info(f'passed resources => {resources}') + + # iterating through resources and adding sites + if len(resources) > 0: + for item in resources: + + if item['type'] == 'site': + try: + sites.append( + Site.objects.get(id=item['id']) + ) + except Exception as e: + logger.warning(e) + + if item['type'] == 'page': + try: + page = Page.objects.get(id=item['id']) + sites.append(page.site) + except Exception as e: + logger.warning(e) + + # iterating through tags + # and adding to sites + if len(tags) > 0: + for tag in tags: + try: + sites += Site.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + try: + tag_pages = Page.objects.filter(tags__contains=[tag]) + sites += [page.site for page in tag_pages] + except Exception as e: + logger.warning(e) + + # grabbing all sites because no + # resources were specified and scope is "account" + if len(resources) == 0 and scope == 'account': + sites = Site.objects.filter(account=account) + + # de-duplicate sites + deduped_sites = [] + seen_site_ids = set() + for site in sites: + if site is None or str(site.id) in seen_site_ids: + continue + seen_site_ids.add(str(site.id)) + deduped_sites.append(site) + sites = deduped_sites + + # record objects for each report + for site in sites: + report_track_id = secrets.token_hex(16) + report_track_map[str(site.id)] = report_track_id + objects.append(_flow_obj( + parent=str(site.id), + obj_id=None, + source_id=str(site.id), + track_id=report_track_id, + status='working' + )) + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'objects': objects, + 'node_status': 'working', + 'message': f'starting {str(len(objects))} site reports | run_id: {flowrun_id}' + }) + + lookback_days = kwargs.get('lookback_days', 7) + report_type = kwargs.get('type', ['issues', 'tests', 'caseruns', 'performance']) + text_color = kwargs.get('text_color', '#24262d') + background_color = kwargs.get('background_color', '#e1effd') + highlight_color = kwargs.get('highlight_color', '#ffffff') + + # create reports for each site + for site in sites: + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + apply_async_in_queue( + create_report, + kwargs={ + 'site_id': str(site.id), + 'lookback_days': lookback_days, + 'type': report_type, + 'text_color': text_color, + 'background_color': background_color, + 'highlight_color': highlight_color, + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'track_id': report_track_map.get(str(site.id)), + '_queue': queue, + }, + queue=queue, + ) + + # update schedule if task_id is not None + update_schedule(task_id=task_id) + + logger.info('Created new site Reports') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def create_auto_cases_bg( + self, + site_id: str=None, + process_id: str=None, + start_url: str=None, + max_cases: int=4, + max_layers: int=5, + configs: dict=settings.CONFIGS + ) -> None: + """ + Generates new `Cases` for the passed site. + + Args: + site_id : str, + process_id : str, + start_url : str, + max_cases : int, + max_layers : int, + configs : dict + + Returns: None + """ + + # checking location + if not check_location(configs.get('location', settings.LOCATION)): + logger.info('Not running due to location param') + return None + + # get objects + site = Site.objects.get(id=site_id) + process = Process.objects.get(id=process_id) + + # get current task and save to process + task_id = str(self.request.id) + process.info = {'task_id': task_id} + process.save() + + # init AutoCaser + AC = AutoCaser( + site=site, + process=process, + start_url=start_url, + configs=configs, + max_cases=max_cases, + max_layers=max_layers, + ) + + # build cases + AC.build_cases() + + logger.info('Built new auto Cases') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def case_pre_run_bg( + self, + case_id: str=None, + process_id: str=None, + ) -> None: + """ + Runs Caser.pre_run() for the passed case_id + + Args: + case_id : str, + process_id : str, + + Returns: None + """ + + # get objects + case = Case.objects.get(id=case_id) + process = Process.objects.get(id=process_id) + + # get current task and save to process + task_id = str(self.request.id) + process.info = {'task_id': task_id} + process.save() + + # init Caser + C = Caser( + case=case, + process=process, + ) + + # build cases + C.pre_run() + + logger.info('Completed Case pre_run') + return None + + + + +@shared_task(bind=True, base=BaseTaskWithRetry) +def run_case( + self, + caserun_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None, + _queue: str=None, + **kwargs, + ) -> None: + """ + Runs a CaseRun. + + Args: + caserun_id : str, + alert_id : str, + flowrun_id : str, + node_index : str + + Returns: None + """ + + account_id = _get_account_id_from_caserun_id(str(caserun_id)) if caserun_id else None + with (account_concurrency_slot(self, account_id=account_id) if account_id else _always_acquired()) as slot: + acquired, rank = slot + if not acquired: + _reschedule_due_to_concurrency(self, rank=rank) + return None + + # get caserun + caserun = CaseRun.objects.get(id=caserun_id) + + # running caserun + Caser( + caserun=caserun, + flowrun_id=flowrun_id, + node_index=node_index + ).run() + + # run alert if requested + if alert_id and alert_id != 'None': + Alerter(alert_id=alert_id, object_id=str(caserun.id)).run_alert() + + logger.info('Ran CaseRun') + return None + + + + +@shared_task +def create_caserun_bg(**kwargs) -> None: + """ + Creates and or runs a CaseRun. + + Args: + caserun_id : str, + resources : list, + tags : list, + scope : str, + account_id : str, + case_id : str, + updates : list, + alert_id : str, + configs : dict, + task_id : str, + flowrun_id : str, + node_index : str + + Returns: None + """ + + # get data + caserun_id = kwargs.get('caserun_id') + case_id = kwargs.get('case_id') + account_id = kwargs.get('account_id') + resources = kwargs.get('resources', []) + tags = kwargs.get('tags', []) + scope = kwargs.get('scope') + updates = kwargs.get('updates') + alert_id = kwargs.get('alert_id') + task_id = kwargs.get('task_id') + configs = kwargs.get('configs', settings.CONFIGS) + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + + # check for redis lock + redis_id = task_id if task_id else secrets.token_hex(8) + lock_name = f"lock:create_caserun_bg_{redis_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # checking location + if not check_location(configs.get('location', settings.LOCATION)): + logger.info('Not running due to location param') + return None + + queue = get_task_queue(kwargs=kwargs) + + # settign defaults + case = None + steps = None + caseruns = [] + sites = [] + objects = [] + + # get case + if case_id: + case = Case.objects.get(id=case_id) + + # update steps + if case: + steps = requests.get(case.steps['url']).json() + for step in steps: + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['status'] = None + + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['status'] = None + + # adding updates + if steps: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + # getting caserun + if caserun_id: + caseruns = [CaseRun.objects.get(id=caserun_id),] + + # creating caserun from case + if caserun_id is None: + + # getting all sites in resources + for item in resources: + if item['type'] == 'site': + try: + sites.append( + Site.objects.get(id=item['id']) + ) + except Exception as e: + logger.warning(e) + + # iterating through tags + # and adding to sites + if len(tags) > 0: + for tag in tags: + + # adding to sites + try: + sites += Site.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + + # add all sites in account if scope == 'account' + if scope == 'account' and len(resources) == 0: + sites = Site.objects.filter(account__id=account_id) + + # iterate through sites + for site in sites: + + # check and increment resource + if check_and_increment_resource(site.account.id, 'caseruns'): + + # create new caserun + caserun = CaseRun.objects.create( + case = case, + title = case.title, + site = site, + user = site.user, + account = site.account, + configs = configs, + steps = steps + ) + + # add to list + caseruns.append(caserun) + + # add to objects + objects.append(_flow_obj( + parent=str(site.id), + obj_id=str(caserun.id), + source_id=str(caserun.id), + track_id=str(caserun.id), + status='working' + )) + + else: + # update flowrun if not able to contiune + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'node_status': 'failed', + 'message': ( + f'❌ case run could not start because this account has reached '+ + f'max_allowed_caseruns for this billing cycle' + ) + }) + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'node_status': 'working', + 'objects': objects + }) + + # iterate through caseruns and run + for caserun in caseruns: + apply_async_in_queue( + run_case, + kwargs={ + 'caserun_id': str(caserun.id), + 'alert_id': alert_id, + 'flowrun_id': flowrun_id, + 'node_index': node_index, + '_queue': queue, + }, + queue=queue, + ) + + # update schedule if task_id is not None + update_schedule(task_id=task_id) + + logger.info('Created CaseRuns') + return None + + + + +@shared_task +def create_flowrun_bg(**kwargs) -> None: + """ + Creates and runs a FlowRun. + + Args: + flow_id : str, + resources : list, + tags : list, + scope : str, + account_id : str, + alert_id : str, + configs : dict, + task_id : str + + Returns: None + """ + + # get data + flow_id = kwargs.get('flow_id') + account_id = kwargs.get('account_id') + resources = kwargs.get('resources', []) + tags = kwargs.get('tags', []) + scope = kwargs.get('scope') + alert_id = kwargs.get('alert_id') + task_id = kwargs.get('task_id') + configs = kwargs.get('configs', settings.CONFIGS) + + # check for redis lock + redis_id = task_id if task_id else secrets.token_hex(8) + lock_name = f"lock:create_flowrun_bg_{redis_id}" + with task_lock(lock_name) as lock_acquired: + + # checking if task is already running + if not lock_acquired: + logger.info('task is already running, skipping execution.') + return None + + # checking location + if not check_location(configs.get('location', 'us')): # not using `settings.LOCATION` for now + logger.info('Not running due to location param') + return None + + # settign defaults + flow = None + sites = [] + + # get flow + if flow_id: + flow = Flow.objects.get(id=flow_id) + + # getting all sites in resources + for item in resources: + if item['type'] == 'site': + try: + sites.append( + Site.objects.get(id=item['id']) + ) + except Exception as e: + logger.info(e) + + # iterating through tags + # and adding to sites + if len(tags) > 0: + for tag in tags: + + # adding to sites + try: + sites += Site.objects.filter(tags__contains=[tag]) + except Exception as e: + logger.warning(e) + + # add all sites in account if scope == 'account' + if scope == 'account' and len(resources) == 0: + sites = Site.objects.filter(account__id=account_id) + + # iterate through sites + for site in sites: + + # check and increment resource + if check_and_increment_resource(site.account.id, 'flowruns'): + + # set flowrun_id + flowrun_id = uuid.uuid4() + + # update nodes + _nodes = flow.nodes + for i in range(len(_nodes)): + _nodes[i]['data']['status'] = 'queued' + _nodes[i]['data']['finalized'] = False + _nodes[i]['data']['time_started'] = None + _nodes[i]['data']['time_completed'] = None + _nodes[i]['data']['alert_id'] = alert_id + _nodes[i]['data']['objects'] = [] + + # updates edges + _edges = flow.edges + for i in range(len(_edges)): + _edges[i]['animated'] = False + _edges[i]['style'] = None + + # create init log + logs = [{ + 'timestamp': timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f'), + 'message': f'system starting up for run_id: {str(flowrun_id)}', + 'step': '1' + },] + + # create flowrun + FlowRun.objects.create( + id = flowrun_id, + flow = flow, + user = flow.user, + account = flow.account, + site = site, + title = flow.title, + nodes = _nodes, + edges = _edges, + logs = logs, + configs = configs + ) + + # update flow with time_last_run + flow = Flow.objects.get(id=flow_id) + flow.time_last_run = timezone.now() + flow.save() + + else: + logger.info('max flowruns reached') + + # update schedule + update_schedule(task_id=task_id) + + logger.info('Created FlowRuns') + return None + + + + +def create_issue( + account_id : str=None, + object_id : str=None, + title : str=None, + details : str=None, + generate : bool=False + ) -> dict: + """ + Creates and `Issue` for each passed obj, using either + passed data or Issuer.build_issue() + + Args: + 'account_id' : str, + 'object_id' : str, + 'title' : str, + 'details' : str, + 'generate' : bool -) -from .models import Log -from django.contrib.auth.models import User -from .utils.driver_p import driver_test -from asgiref.sync import async_to_sync -import asyncio + Returns: + 'message' : str, + 'success' : bool + """ + # set defaults + message = '' + success = True + affected = {} + trigger = {} + issue = None -logger = get_task_logger(__name__) + # get object from object_id + obj = get_obj(object_id=object_id) + # check for success + if not obj.get('success'): + message = f'❌ object not found - unable to create issue for {object_id}' + success = False + return {'message': message, 'success': success, 'issue': issue} + # check for 'generate' + if generate: + # build Issue using Issuer().build_issue() + try: + issue = Issuer( + scan = obj.get('obj') if obj.get('obj_type') == 'Scan' else None, + test = obj.get('obj') if obj.get('obj_type') == 'Test' else None, + caserun = obj.get('obj') if obj.get('obj_type') == 'CaseRun' else None, + threshold = obj.get('obj').threshold if obj.get('obj_type') == 'Test' else 75 + ).build_issue() -@shared_task -def test_pupeteer(): - asyncio.run(driver_test()) - logger.info('Tested pupeteer instalation') + # build messge + message = f'created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}' + success = True + + except Exception as e: + logger.info(e) + # build messge + message = f'❌ generation failed - unable to create issue for {object_id}' + success = False + # check for manual creation + if not generate: + + # create trigger + trigger = { + 'type' : obj.get('obj_type').lower(), + 'id' : str(obj.get('obj').id) + } -@shared_task -def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs): - create_site_task(site_id, scan_id, configs) - logger.info('Created scan of new site') + # create affected + affected = { + 'type' : 'site' if obj.get('obj_type') == 'CaseRun' else 'page', + 'id' : str(obj.get('obj').site.id) if obj.get('obj_type') == 'CaseRun' else str(obj.get('obj').page.id), + 'str' : obj.get('obj').site.site_url if obj.get('obj_type') == 'CaseRun' else obj.get('obj').page.page_url + } + # get account & secrets + account = Account.objects.get(id=account_id) + secrets = Secret.objects.filter(account=account) -@shared_task -def create_scan_bg( - scan_id=None, - site_id=None, - type=['full'], - automation_id=None, - configs=None, - tags=None, - *args, - **kwargs, - ): - create_scan_task( - scan_id, - site_id, - type, - automation_id, - configs, - tags, - ) - logger.info('Created new scan of site') + # transpose data + title = transpose_data(title, obj.get('obj'), secrets) + details = transpose_data(details, obj.get('obj'), secrets) + # build Issue + issue = Issue.objects.create( + account = account, + title = title, + details = details, + labels = [], + trigger = trigger, + affected = affected + ) + # build messge + message = f'created new issue for {obj.get('obj_type').lower()} | issue_id {issue.id}' + success = True + + # return data + data = { + 'message' : message, + 'success' : success, + 'issue' : issue + } + return data -@shared_task -def run_html_and_logs_bg(scan_id=None, *args, **kwargs): - run_html_and_logs_task(scan_id) - logger.info('ran html & logs component') @shared_task -def run_vrt_bg(scan_id=None, *args, **kwargs): - run_vrt_task(scan_id) - logger.info('ran vrt component') +def create_issue_bg( + account_id: str=None, + objects: list=None, + title: str=None, + details: str=None, + generate: bool=True, + flowrun_id: str=None, + node_index: str=None + ) -> None: + """ + Runs create_issue for each passed `object` + + Args: + 'account_id' : str, + 'objects' : list, + 'title' : str, + 'details' : str, + 'generate' : bool, + 'flowrun_id' : str, + 'node_index' : str, + + Returns: None + """ + + # create objects list for flowrun + obj_list = [] + for o in objects: + obj_list.append(_flow_obj( + parent=o.get('id'), + obj_id=None, + source_id=o.get('source_id', o.get('id')), + track_id=o.get('track_id'), + status='working' + )) + + # update flowrun if requested + if flowrun_id and flowrun_id != 'None': + # update flowrun + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': f'building {len(obj_list)} Issues | run_id: {flowrun_id}', + 'objects': obj_list + }) + + # interating through objects + for obj in objects: + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # run create_issue + source_object_id = obj.get('source_id') or obj.get('id') + resp = create_issue( + account_id=account_id, + object_id=source_object_id, + title=title, + details=details, + generate=generate + ) + + if flowrun_id and flowrun_id != 'None': + # update flowrun + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': resp.get('message'), + 'objects': [_flow_obj( + parent=obj.get('id'), + obj_id=str(resp.get('issue').id) if resp.get('success') else None, + source_id=( + str(resp.get('issue').id) + if resp.get('success') + else obj.get('source_id', obj.get('id')) + ), + track_id=obj.get('track_id'), + status='passed' if resp.get('success') else 'failed' + )] + }) + + logger.info('created issues') + return None -@shared_task -def run_lighthouse_bg(scan_id=None, *args, **kwargs): - run_lighthouse_task(scan_id) - logger.info('ran lighthouse component') @shared_task -def run_yellowlab_bg(scan_id=None, *args, **kwargs): - run_yellowlab_task(scan_id) - logger.info('ran yellowlab component') +def delete_site_s3_bg(site_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed site + + Args: + 'site_id': str + + Returns: None + """ + + # deleting s3 objects + try: + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() + except: + pass + + logger.info('Deleted site s3 objects') + return None @shared_task -def create_test_bg( - test_id=None, - site_id=None, - automation_id=None, - configs=None, - type=['full'], - index=None, - pre_scan=None, - post_scan=None, - tags=None, - *args, - **kwargs, - ): - create_test_task( - test_id, - site_id, - automation_id, - configs, - type, - index, - pre_scan, - post_scan, - tags - ) - logger.info('Created new test of site') +def delete_page_s3_bg(page_id: str, site_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed page + + Args: + 'site_id': str, + 'page_id': str + + Returns: None + """ + + # deleting s3 objects + try: + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete() + except: + pass + + logger.info('Deleted page s3 objects') + return None + + + @shared_task -def create_report_bg(site_id=None, automation_id=None, *args, **kwargs): - create_report_task(site_id, automation_id) - logger.info('Created new report of site') +def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed scan + + Args: + 'scan_id': str, + 'site_id': str, + 'page_id': str + + Returns: None + """ + + # deleting s3 objects + try: + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete() + except: + pass + + logger.info('Deleted scan s3 objects') + return None + + @shared_task -def delete_site_s3_bg(site_id, *args, **kwargs): - delete_site_s3(site_id) - logger.info('Deleted site s3 objects') +def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed test + + Args: + 'test_id': str, + 'site_id': str, + 'page_id': str + + Returns: None + """ + + # deleting s3 objects + try: + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete() + except: + pass + + logger.info('Deleted test s3 objects') + return None + + @shared_task -def delete_testcase_s3_bg(testcase_id, *args, **kwargs): - delete_testcase_s3(testcase_id) - logger.info('Deleted testcase s3 objects') +def delete_caserun_s3_bg(caserun_id: str) -> None: + """ + Deletes the directory in s3 bucked associated + with passed test + + Args: + 'caserun_id': str, + + Returns: None + """ + + # deleting s3 objects + try: + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/caserun/{caserun_id}/')).delete() + except: + pass + + logger.info('Deleted caserun s3 objects') + return None + + @shared_task -def delete_report_s3_bg(report_id, *args, **kwargs): - delete_report_s3(report_id) +def delete_report_s3_bg(report_id: str) -> None: + """ + Deletes the file in s3 bucked associated + with passed report + + Args: + 'report_id': str, + + Returns: None + """ + + try: + report = Report.objects.get(id=report_id) + site = report.site or (report.page.site if report.page else None) + if site is None: + logger.info('No site found for report; skipping report s3 delete') + return None + + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/reports/{report_id}.pdf')).delete() + except: + pass + logger.info('Deleted Report pdf in s3') + return None + + + + +@shared_task +def delete_case_s3_bg(case_id: str) -> None: + """ + Deletes the file in s3 bucked associated + with passed case_id + + Args: + 'case_id': str, + + Returns: None + """ + + # deleting s3 objects + try: + bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME) + bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete() + except: + pass + + logger.info('Deleted Case step data in s3') + return None + + @shared_task -def purge_logs(username=None, *args, **kwargs): +def purge_logs(username: str=None) -> None: + """ + Deletes all `Logs` associated with the passed "username". + If "username" is None, deletes all `Logs`. + + Args: + 'username': str + + Returns: None + """ + + # delete logs if username: user = User.objects.get(username=username) Log.objects.filter(user=user).delete() @@ -145,60 +3250,788 @@ def purge_logs(username=None, *args, **kwargs): Log.objects.all().delete() logger.info('Purged logs') + return None + + @shared_task -def create_testcase_bg( - testcase_id=None, - site_id=None, - case_id=None, - updates=None, - automation_id=None, - configs=None, - type=None, - *args, - **kwargs, - ): - create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id) - logger.info('Ran full testcase') +def reset_account_usage(account_id: str=None) -> None: + """ + Loops through each active `Account`, checks to see + if timezone.now() is the start of the + next billing cycle, and resets `Account.usage` + + Args: + 'account_id': (OPTIONAL) + + Returns: None + """ + + # set defaults + stripe.api_key = settings.STRIPE_PRIVATE + today = timezone.now() + + # get accounts + if account_id: + accounts = [Account.objects.get(id=account_id)] + else: + accounts = Account.objects.filter(active=True) + + # helper reset method + def reset_usage(account): + account.usage.update({ + 'scans': 0, + 'tests': 0, + 'caseruns': 0, + 'flowruns': 0 + }) + account.meta = account.meta or {} + account.meta['last_usage_reset'] = today.isoformat() + account.save() + logger.info(f'Reset usage for account: {account.name}') + + for account in accounts: + + # defaults + needs_reset = False + + # get last reset data + last_reset_str = (account.meta or {}).get('last_usage_reset') + last_reset = None + + # format last reset date + try: + if last_reset_str: + last_reset = datetime.fromisoformat(last_reset_str.replace("Z", "")) + # if last_reset is naive, make it timezone-aware + if timezone.is_naive(last_reset): + last_reset = timezone.make_aware(last_reset) + except: + last_reset = None + + # check stripe sub if paying account + if account.type != 'free' and account.sub_id: + try: + # get last invoice date + sub = stripe.Subscription.retrieve(account.sub_id) + sub_reset_date = timezone.make_aware( + datetime.fromtimestamp(sub.current_period_start) + ) + + # make last_reset same as sub_reset day if none exists + last_reset = sub_reset_date if not last_reset else last_reset + + # check if invoice is 30 days or older || last_reset_date is 30 days or older + if (today - sub_reset_date).days >= 30 or (today - last_reset).days >= 30: + needs_reset = True + + except stripe.error.StripeError as e: + logger.info(f'Stripe error for account {account.id}: {e}') + + # reset free & selfhost accounts + elif account.type in ['free', 'selfhost']: + if not last_reset or (today - last_reset).days >= 30: + needs_reset = True + + # reset passed account_id + if account_id: + needs_reset = True + + # trigger reset + if needs_reset: + reset_usage(account) + + return None + + + + +@shared_task +def update_sub_price(account_id: str=None, sites_allowed: int=None) -> None: + """ + Update price for existing stripe Subscription + based on new `Account.usage.sites_allowed` + + Args: + 'account_id' : (REQUIRED) + 'sites_allowed' : (OPTIONAL) + + Returns: None + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get account + account = Account.objects.get(id=account_id) + + # set new sites_allowed + if sites_allowed is not None: + account.usage['sites_allowed'] = sites_allowed + account.save() + + # get sites_allowed + if sites_allowed is None: + sites_allowed = account.usage['sites_allowed'] + + # get account coupon + discount = 0 + if account.meta.get('coupon'): + discount = account.meta['coupon']['discount'] + + # calculate + price = ( + ( + 54.444 * (sites_allowed ** 0.4764) + ) * 100 + ) + + # apply discount + price = price - (price * discount) + + # update for interval + price_amount = round(price if account.interval == 'month' else (price * 10)) + + # create new Stripe Price + price = stripe.Price.create( + product=account.product_id, + unit_amount=price_amount, + currency='usd', + recurring={'interval': account.interval,}, + ) + + # update Stripe Subscription + sub = stripe.Subscription.retrieve(account.sub_id) + stripe.Subscription.modify( + account.sub_id, + cancel_at_period_end=False, + pause_collection='', + proration_behavior='create_prorations', + items=[{ + 'id': sub['items']['data'][0].id, + 'price': price.id, + }], + expand=['latest_invoice.payment_intent'], + ) + + # updating price defaults and archiving old price + stripe.Product.modify(account.product_id, default_price=price,) + stripe.Price.modify(account.price_id, active=False) + + # update account with new info + account.price_id = price.id + account.price_amount = price_amount + account.usage['sites'] = sites_allowed + account.usage['scans_allowed'] = (sites_allowed * 200) + account.usage['tests_allowed'] = (sites_allowed * 200) + account.usage['caseruns_allowed'] = (sites_allowed * 10) + account.usage['flowruns_allowed'] = (sites_allowed * 10) + account.save() + + logger.info(f'new price -> {price_amount}') + + # return + return None + + + + +@shared_task +def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None: + """ + Deletes all `Tests`, `Scans`, `CaseRuns`, `FlowRuns`, + `Logs`, `Issues`, and `Processes` that have reached expiry + + Args: + account_id : str, + days_to_live : int + + Returns: None + """ + + # calculate max dates + max_date = timezone.now() - timedelta(days=days_to_live) + max_proc_date = timezone.now() - timedelta(days=1) + + # scope resources to account if requested + if account_id is not None: + tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date) + scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date) + caseruns = CaseRun.objects.filter(account__id=account_id, time_created__lte=max_date) + flowruns = FlowRun.objects.filter(account__id=account_id, time_created__lte=max_date) + processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date) + issues = Issue.objects.filter(account__id=account_id, time_created__lte=max_proc_date) + + # get all old Logs + members = Member.objects.filter(account__id=account_id) + logs = [] + for member in members: + logs += Log.objects.filter(user=member.user, time_created__lte=max_proc_date) + + # get all resoruces if no account_id + else: + tests = Test.objects.filter(time_created__lte=max_date) + scans = Scan.objects.filter(time_created__lte=max_date) + caseruns = CaseRun.objects.filter(time_created__lte=max_date) + flowruns = FlowRun.objects.filter(time_created__lte=max_date) + processes = Process.objects.filter(time_created__lte=max_proc_date) + logs = Log.objects.filter(time_created__lte=max_proc_date) + issues = Issue.objects.filter(time_created__lte=max_proc_date) + + # delete each resource in each type + for test in tests: + delete_test_s3_bg.delay(test.id, test.site.id, test.page.id) + test.delete() + for scan in scans: + delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id) + scan.delete() + for caserun in caseruns: + delete_caserun_s3_bg.delay(caserun.id) + caserun.delete() + for flowrun in flowruns: + flowrun.delete() + for process in processes: + process.delete() + for issue in issues: + issue.delete() + for log in logs: + log.delete() + + logger.info('Cleaned up resources') + return None + + + + +@shared_task +def data_retention() -> None: + """ + Helper task for looping through each account and deleting old resources using + delete_old_resources() + + Returns: None + """ + + # get all accounts + accounts = Account.objects.all() + + # loop through each account + for account in accounts: + + # delete old resources + delete_old_resources.delay( + account_id=account.id, + days_to_live=account.usage['retention_days'] + ) + + logger.info('Requested resource cleanup') + return None + + + + +@shared_task +def delete_admin_sites(days_to_live: int=1) -> None: + """ + Delete all admin sites which are older + than 'days_to_live' + + Args: + 'days_to_live': int + + Returns: None + """ + + # calculate max date + max_date = timezone.now() - timedelta(days=days_to_live) + + # filter sites by max_date and admin + sites = Site.objects.filter(time_created__lte=max_date, user__username='admin') + + # delete each site + for site in sites: + delete_site_s3_bg.delay(site.id) + site.delete() + + logger.info('Cleaned up admin sites') + return None + + + + +@shared_task +def create_prospect(user_email: str=None) -> None: + """ + Sends an API request to Cursion Landing which + creates a new `Prospect` + + Args: + 'user_email': str + + Returns: None + """ + + if settings.MODE == 'selfhost': + logger.info('not running because of selfhost mode') + return None + + # get user by id + user = User.objects.get(email=user_email) + phone = None + if Member.objects.filter(user=user).exists(): + member = Member.objects.get(user=user) + phone = member.phone + + # get account by user + account = Account.objects.get(user=user) + + # determinig user's 'status' + if account.type == 'free': + if Site.objects.filter(account=account).exists(): + _status = 'warm' # account has one site onboarded + else: + _status = 'cold' # account is free but no site onboarded + if account.type != 'free': + if account.active: + _status = 'customer' # account is active and paid + else: + _status = 'warm' # account is paused and paid + if account.type == 'new': + _status = 'cold' # account has not onboarded + if account.type == 'selfhost': + _status = 'customer' # account is active and paid + + # setup configs + url = f'{settings.LANDING_URL_ROOT}/ops/prospect' + headers = { + "content-type": "application/json", + "Authorization" : f'Token {settings.LANDING_API_KEY}' + } + data = { + 'first_name': str(user.first_name), + 'last_name': str(user.last_name), + 'email': str(user.email), + 'phone': phone, + 'license_key': str(account.license_key), + 'info': account.info, + 'status': _status, + 'source': 'app', + } + + try: + # send the request + res = requests.post( + url=url, + headers=headers, + data=json.dumps(data) + ).json() + + success = True + message = res + + except Exception as e: + success = False + message = e + + # format response + data = { + 'success': success, + 'message': message + } + + logger.info(f'Sent Prospect creation request -> {data}') + return None + + + + +@shared_task +def create_report_export_bg(report_id: str=None, email: str=None, first_name: str=None) -> None: + """ + Creates and exports a Cursion landing report + + Args: + report_id : str, + email : str, + first_name : str + + Returns: None + """ + + # create and export + data = create_and_send_report_export( + report_id=report_id, + email=email, + first_name=first_name + ) + + logger.info(f'Created and sent report export -> {data}') + return None + + + + +@shared_task +def send_invite_link_bg(member_id: str) -> None: + """ + Sends an invite link to the requested member + + Args: + 'member_id': str + + Returns: None + """ + + # get member + member = Member.objects.get(id=member_id) + + # send invite + send_invite_link(member) + + logger.info('Sent invite') + return None + + + + +@shared_task +def send_remove_alert_bg(member_id: str) -> None: + """ + Sends a 'removed' email to the requested member + + Args: + 'member_id': str + + Returns: None + """ + + # get member + member = Member.objects.get(id=member_id) + + # send email + send_remove_alert(member) + + logger.info('Sent remove alert') + return None + + + + +@shared_task +def send_phone_bg( + account_id: str=None, + objects: list=None, + phone_number: str=None, + body: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> dict: + """ + Run `Alerts.send_phone` as a backgroud task + + Args: + 'account_id' : str, + 'objects' : list, + 'phone_number' : str, + 'body' : str, + 'flowrun_id' : str, + 'node_index' : str, + + Returns: + None + """ + + # interating through objects + for obj in objects: + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # run send_phone + source_object_id = obj.get('source_id') or obj.get('id') + resp = send_phone( + account_id=account_id, + object_id=source_object_id, + phone_number=phone_number, + body=body, + ) + + if flowrun_id and flowrun_id != 'None': + # update flowrun + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': resp.get('message'), + 'objects': [_flow_obj( + parent=obj.get('parent'), + obj_id=obj.get('id'), + source_id=obj.get('source_id', obj.get('id')), + track_id=obj.get('track_id'), + status='passed' if resp.get('success') else 'failed' + )] + }) + + logger.info('sent phone message') + return None + + + + +@shared_task +def send_slack_bg( + account_id: str=None, + objects: list=None, + body: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> dict: + """ + Run `Alerts.send_slack` as a backgroud task + + Args: + 'account_id' : str, + 'objects' : list, + 'body' : str, + 'flowrun_id' : str, + 'node_index' : str, + + Returns: None + """ + + # interating through objects + for obj in objects: + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # run send_slack + source_object_id = obj.get('source_id') or obj.get('id') + resp = send_slack( + account_id=account_id, + object_id=source_object_id, + body=body, + ) + + if flowrun_id and flowrun_id != 'None': + # update flowrun + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': resp.get('message'), + 'objects': [_flow_obj( + parent=obj.get('parent'), + obj_id=obj.get('id'), + source_id=obj.get('source_id', obj.get('id')), + track_id=obj.get('track_id'), + status='passed' if resp.get('success') else 'failed' + )] + }) + + logger.info('sent slack message') + return None + + + + +@shared_task +def send_email_bg( + account_id: str=None, + objects: list=None, + message_obj: dict=None, + flowrun_id: str=None, + node_index: str=None + ) -> dict: + """ + Run `Alerts.sendgrid_email` as a backgroud task + + Args: + 'account_id' : str, + 'objects' : list, + 'message_obj' : dict, + 'flowrun_id' : str, + 'node_index' : str, + + + Returns: + None + """ + + # interating through objects + for obj in objects: + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # run sendgrid_email + source_object_id = obj.get('source_id') or obj.get('id') + resp = sendgrid_email( + account_id=account_id, + object_id=source_object_id, + message_obj=message_obj, + ) + + if flowrun_id and flowrun_id != 'None': + # update flowrun + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': resp.get('message'), + 'objects': [_flow_obj( + parent=obj.get('parent'), + obj_id=obj.get('id'), + source_id=obj.get('source_id', obj.get('id')), + track_id=obj.get('track_id'), + status='passed' if resp.get('success') else 'failed' + )] + }) + + logger.info('sent email message') + return None + + + + +@shared_task +def send_webhook_bg( + account_id: str=None, + objects: list=None, + request_type: str=None, + url: str=None, + headers: str=None, + payload: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> dict: + """ + Run `Alerts.sendgrid_email` as a backgroud task + + Args: + 'account_id' : str, + 'objects' : list, + 'request_type' : str, + 'url' : str, + 'headers' : str, + 'payload' : str, + 'flowrun_id' : str, + 'node_index' : str, + + Returns: None + """ + + # interating through objects + for obj in objects: + + # sleeping random for DB + time.sleep(random.uniform(2, 6)) + + # run sendgrid_email + source_object_id = obj.get('source_id') or obj.get('id') + try: + resp = send_webhook( + account_id=account_id, + object_id=source_object_id, + request_type=request_type, + url=url, + headers=headers, + payload=payload + ) + except Exception as e: + resp = { + 'success': False, + 'message': str(e) + } + + if flowrun_id and flowrun_id != 'None': + # update flowrun + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': resp.get('message'), + 'objects': [_flow_obj( + parent=obj.get('parent'), + obj_id=obj.get('id'), + source_id=obj.get('source_id', obj.get('id')), + track_id=obj.get('track_id'), + status='passed' if resp.get('success') else 'failed' + )] + }) + + logger.info('sent webhook message') + return None @shared_task def migrate_site_bg( - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - plugin_name, - wait_time, - process_id, - driver, - *args, - **kwargs - ): - migrate_site_task( - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - plugin_name, - wait_time, - process_id, - driver, + login_url: str, + admin_url: str, + username: str, + password: str, + email_address: str, + destination_url: str, + sftp_address: str, + dbname: str, + sftp_username: str, + sftp_password: str, + plugin_name: str, + wait_time: int, + process_id: str, + driver: str, + ) -> None: + """ + Runs the WP site migration process. + + Args: + login_url: str, + admin_url: str, + username: str, + password: str, + email_address: str, + destination_url: str, + sftp_address: str, + dbname: str, + sftp_username: str, + sftp_password: str, + plugin_name: str, + wait_time: int, + process_id: str, + driver: str, + + Returns: None + """ + + # init wordpress for selenium + wp = W( + login_url=login_url, + admin_url=admin_url, + username=username, + password=password, + email_address=email_address, + destination_url=destination_url, + sftp_address=sftp_address, + dbname=dbname, + sftp_username=sftp_username, + sftp_password=sftp_password, + wait_time=wait_time, + process_id=process_id, + ) - logger.info('Finished Migration') \ No newline at end of file + # login + wp_status = wp.login() + # adjust lang + wp_status = wp.begin_lang_check() + # install plugin + wp_status = wp.install_plugin(plugin_name=plugin_name) + # launch migration + wp_status = wp.launch_migration() + # run migration + wp_status = wp.run_migration() + # re adjust lang + # wp_status = wp.end_lang_check() + + + logger.info('Finished Migration') + return None diff --git a/app/api/templates/api/alert_no_button.html b/app/api/templates/api/alert_no_button.html index 20e8260e..6dbc4813 100644 --- a/app/api/templates/api/alert_no_button.html +++ b/app/api/templates/api/alert_no_button.html @@ -137,12 +137,12 @@
- Scanerr, San Antonio TX + Cursion, San Antonio TX
- Powered by Scanerr. + Powered by Cursion.
diff --git a/app/api/templates/api/alert_with_button.html b/app/api/templates/api/alert_with_button.html index 13c30107..2b163e4d 100644 --- a/app/api/templates/api/alert_with_button.html +++ b/app/api/templates/api/alert_with_button.html @@ -144,12 +144,12 @@
- Scanerr, San Antonio TX + Cursion, San Antonio TX
- Powered by Scanerr. + Powered by Cursion.
diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html index 6ba3c45f..30719264 100644 --- a/app/api/templates/api/automation_email.html +++ b/app/api/templates/api/automation_email.html @@ -152,12 +152,12 @@
- Scanerr, San Antonio TX + Cursion, San Antonio TX
- Powered by Scanerr. + Powered by Cursion.
diff --git a/app/api/tests.py b/app/api/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/app/api/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/app/api/urls.py b/app/api/urls.py index 9df63e49..39d9edea 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -1,11 +1,13 @@ from .v1 import urls as v1_urls from django.urls import path, include +from django.views.generic.base import RedirectView urlpatterns = [ path('v1/', include(v1_urls)), + path('', RedirectView.as_view(url='v1/auth/', permanent=False)) ] diff --git a/app/api/utils/agent.py b/app/api/utils/agent.py new file mode 100644 index 00000000..e30cab39 --- /dev/null +++ b/app/api/utils/agent.py @@ -0,0 +1,106 @@ +from rest_framework.authtoken.models import Token +from django.utils import timezone +from ..models import Chat +from cursion import settings +from openai import OpenAI + + + + + + +class Agent(): + """ + Generate new respones for the passed 'Chat'. + + Args: + 'chat_id': str + + Use `Agent.respond()` to generate a response to the + latest `chat.message` + + Returns: + None + """ + + + def __init__(self, chat_id: str=None) -> None: + self.chat = Chat.objects.get(id=chat_id) + self.llm = OpenAI(api_key=settings.GPT_API_KEY) + + + + + def respond(self) -> object: + """ + Using the latest entry in `chat.message`, and chat + history sends a request to the self.llm and + appends the response to chat.message. + + Args: + None + + Returns: + None + """ + + # get user's token + token_obj = Token.objects.get(user=self.chat.user) if self.chat else None + + # format chat history + history_parts = [] + + # iterate through each message and build chat context + for m in self.chat.messages: + author = m.get('author') or m.get('user') + role = 'assistant' if author == 'agent' else 'user' + text = m.get('text', '').strip() + name = 'Agent' if author == 'agent' else self.chat.user.first_name + history_parts.append(f'[{role.upper()} — {name}]\n{text}') + + # concat into string + chat_history = '\n\n'.join(history_parts) + + # full prompt + input_string = ( + 'BACKGROUND CONTEXT:\n' + 'You are a Software Quality Assurance Engineer.\n' + # 'Please reference https://docs.cursion.dev for documentation about the Cursion Platform.\n' + 'If necessary, call Cursion MCP tools to complete the task.\n' + 'If responding with `Site`, `Page`, `Scan`, `Test`, `Case`, `CaseRun`, `Flow`, or `FlowRun` objects, ' + 'include their URL formatted like so: ' + f'"{settings.CLIENT_URL_ROOT}//"\n' + '\n\n' + f'CHAT HISTORY:\n{chat_history}' + ) + + # build mcp url + mcp_base = (settings.MCP_URL_ROOT or '').rstrip('/') + mcp_url = mcp_base if mcp_base.endswith('/sse') else f'{mcp_base}/sse' + + # call llm + response = self.llm.responses.create( + model='gpt-5-mini', + input=input_string, + tools=[{ + 'type' : 'mcp', + 'server_label' : 'cursion-mcp', + 'server_url' : mcp_url, + 'require_approval' : 'never', + 'authorization' : f'Token {token_obj.key}' + }] + ) + + # add response message + messages = self.chat.messages + messages.append({ + 'author': 'agent', + 'time_created': str(timezone.now()), + 'text': response.output_text + }) + self.chat.messages = messages + self.chat.save() + + # return updated chat + return self.chat + diff --git a/app/api/utils/alerter.py b/app/api/utils/alerter.py new file mode 100644 index 00000000..f8607a49 --- /dev/null +++ b/app/api/utils/alerter.py @@ -0,0 +1,260 @@ +from ..models import * +from .alerts import * +import re + + + + + + +class Alerter(): + """ + Build and execute `Alert` logic generated by a user. + + Args: + 'alert_id' : str, + 'object_id' : str, + 'expressions' : list + + - Use `Alerter.run_alert()` to run an `Alert` + - Use `Alerter.get_object()` to set self.object + - Use `Alerter.build_expressions()` to get self.exp_string + + Returns: None + """ + + + def __init__( + self, + alert_id: str=None, + object_id: str=None, + expressions: list=[], + task_type: str=None + ): + + self.alert = Alert.objects.get(id=alert_id) if (alert_id and alert_id != 'None') else None + self.expressions = self.alert.expressions if self.alert else expressions + self.task_type = self.alert.schedule.task_type if self.alert else task_type + self.object_id = object_id + self.exp_string = '1 == 1' + self.act_string = '' + self.object = None + self.use_exp = True + + + + + def get_object(self) -> bool: + """ + Tries to get the focus object from self.object_id - if found + will set self.object and self.use_exp + + Returns: None or object + """ + + if self.task_type == 'scan': + try: + self.object = Scan.objects.get(id=self.object_id) + self.use_exp = True + return self.object + except: + return None + + elif self.task_type == 'test': + try: + self.object = Test.objects.get(id=self.object_id) + self.use_exp = True + return self.object + except: + return None + + elif self.task_type == 'report': + try: + self.object = Report.objects.get(id=self.object_id) + self.use_exp = False + return self.object + except: + return None + + elif self.task_type == 'caserun' or self.task_type == 'case': + try: + self.object = CaseRun.objects.get(id=self.object_id) + self.use_exp = True + return self.object + except: + return None + + elif self.task_type == 'flowrun' or self.task_type == 'flow': + try: + self.object = FlowRun.objects.get(id=self.object_id) + self.use_exp = True + return self.object + except: + return None + + else: + return None + + + + + def build_expressions(self) -> None: + """ + Loop through the self.expressions + and rebuilds into self.exp_string + + Returns: self.exp_string + """ + + # begin iteration + exp_list = [] + for expression in self.expressions: + + # set defaults + exp = None + data_type = None + operator = ' == ' + joiner = '' + data_type = 'obj.status' + value = f"str('{str(expression['value'])}')" + non_float_types = ['caserun_status', 'test_status', 'flowrun_status'] + + # get comparison value + if expression['data_type'] not in non_float_types: + try: + value = float(re.search(r'-?\d+(?:\.\d+)?', str(expression['value'])).group()) + except: + value = None + + # get operator + if '>=' in expression['operator']: + operator = ' >= ' + elif '<=' in expression['operator']: + operator = ' <= ' + else: + operator = ' == ' + + # get joiner + if 'and' in expression['joiner']: + joiner = ' and ' + elif 'or' in expression['joiner']: + joiner = ' or ' + else: + joiner = '' + + # get data_type translation + definition = get_definition(expression['data_type']) + if definition: + raw_value = definition['value'] + data_type = f'({raw_value} if {raw_value} else 0)' + + # building exp if not defined + if exp is None: + exp = f'{joiner}{data_type}{operator}{value}' + + # adding exp to exp_list + exp_list.append(exp) + + # build expression string + self.exp_string = ' '.join(exp_list) + + # return exp_string + return self.exp_string + + + + + def build_actions(self) -> None: + """ + Loop through the alert.actions + and rebuilds into self.act_string + + Returns: self.act_string + """ + + # defaults + act_list = [] + + # begin iteration + for action in self.alert.actions: + + if 'slack' in action['action_type']: + action_type = str( + f"\n print('sending slack alert')" + + f"\n alert_slack(alert_id='{str(self.alert.id)}'," + + f" object_id='{str(self.object_id)}')" + ) + + if 'email' in action['action_type']: + action_type = str( + f"\n print('sending email alert')" + + f"\n alert_email(email='{action['email']}'," + + f" alert_id='{str(self.alert.id)}'," + + f" object_id='{str(self.object_id)}')" + ) + + if type(self.object).__name__ == 'Report': + action_type = str( + f"\n print('sending report email')" + + f"\n alert_report_email(email='{action['email']}'," + + f" alert_id='{str(self.alert.id)}'," + + f" object_id='{str(self.object_id)}')" + ) + + if 'phone' in action['action_type']: + action_type = str( + f"\n print('sending phone alert')" + + f"\n alert_phone(phone_number='{action['phone']}'," + + f" alert_id='{str(self.alert.id)}'," + + f" object_id='{str(self.object_id)}')" + ) + + # adding action to act_list + act_list.append(action_type) + + # build string + self.act_string = ''.join(act_list) + + # return formated actions string + return self.act_string + + + + + def run_alert(self) -> None: + + # get object data + self.get_object() + + # if obj was retrieved + if self.object: + + # setting obj for defnitions data + obj = self.object + + # build expressions + if self.use_exp: + self.build_expressions() + + # build actions + self.build_actions() + + # building final exec str + alert_logic = f'if {self.exp_string}:{self.act_string}' + + # executing alert logic + exec(alert_logic) + + return None + + + + + + + + + + + + \ No newline at end of file diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py index fb4f68df..df38b935 100644 --- a/app/api/utils/alerts.py +++ b/app/api/utils/alerts.py @@ -1,326 +1,181 @@ -from django.core.mail import send_mail, send_mass_mail from django.contrib.auth.models import User -from django.template.loader import render_to_string -from datetime import date -import os, operator, json, requests, uuid -from django.utils.html import strip_tags -from django.contrib.auth.models import User -from rest_framework.response import Response -from ..models import * +from rest_framework_simplejwt.tokens import RefreshToken from twilio.rest import Client from slack_sdk.web import WebClient from slack_sdk.errors import SlackApiError from sendgrid import SendGridAPIClient -from sendgrid.helpers.mail import Mail, From, To -from scanerr import settings +from sendgrid.helpers.mail import Mail, Subject, Content, From +from ..models import * +from cursion import settings +from .definitions import get_definition, definitions +from datetime import date +from cryptography.fernet import Fernet +import os, json, requests, uuid, re -def create_exp_str(item, automation, is_email=False): - exp_list = [] +def send_reset_link(email: str=None) -> dict: + """ + Sends a reset password email to the User with + the passed 'email' + + Args: + 'email': str - for e in automation.expressions: - if 'test_score' in e['data_type']: - data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t' - elif 'current_health' in e['data_type']: - data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t' - elif 'health' in e['data_type']: - data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t' - # LH test data - elif 'current_lighthouse_average' in e['data_type']: - data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t' - elif 'seo_delta' in e['data_type']: - data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t' - elif 'pwa_delta' in e['data_type']: - data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t' - elif 'crux_delta' in e['data_type']: - data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t' - elif 'best_practices_delta' in e['data_type']: - data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t' - elif 'performance_delta' in e['data_type']: - data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t' - elif 'accessibility_delta' in e['data_type']: - data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t' - # LH scan data - elif 'lighthouse_average' in e['data_type']: - data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t' - elif 'seo' in e['data_type']: - data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t' - elif 'pwa' in e['data_type']: - data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t' - elif 'crux' in e['data_type']: - data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t' - elif 'best_practices' in e['data_type']: - data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t' - elif 'performance' in e['data_type']: - data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t' - elif 'accessibility' in e['data_type']: - data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t' - - + Returns: + 'success': bool + """ - # yellowlab test data - elif 'current_yellowlab_average' in e['data_type']: - data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t' - elif 'pageWeight_delta' in e['data_type']: - data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t' - elif 'requests_delta' in e['data_type']: - data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t' - elif 'domComplexity_delta' in e['data_type']: - data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t' - elif 'javascriptComplexity_delta' in e['data_type']: - data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t' - elif 'badJavascript_delta' in e['data_type']: - data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t' - elif 'jQuery_delta' in e['data_type']: - data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t' - elif 'cssComplexity_delta' in e['data_type']: - data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t' - elif 'badCSS_delta' in e['data_type']: - data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t' - elif 'fonts_delta' in e['data_type']: - data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t' - elif 'serverConfig_delta' in e['data_type']: - data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t' - - # yellowlab scan data - elif 'yellowlab_average' in e['data_type']: - data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t' - elif 'pageWeight' in e['data_type']: - data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t' - elif 'requests' in e['data_type']: - data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t' - elif 'domComplexity' in e['data_type']: - data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t' - elif 'javascriptComplexity' in e['data_type']: - data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t' - elif 'badJavascript' in e['data_type']: - data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t' - elif 'jQuery' in e['data_type']: - data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t' - elif 'cssComplexity' in e['data_type']: - data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t' - elif 'badCSS' in e['data_type']: - data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t' - elif 'fonts' in e['data_type']: - data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t' - elif 'serverConfig' in e['data_type']: - data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t' - - elif 'avg_image_score' in e['data_type']: - data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t' - elif 'image_scores' in e['data_type']: - data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' - - elif 'logs' in e['data_type']: - data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t' + # check if User exists + if User.objects.filter(email=email).exists(): + # build email data + user = User.objects.get(email=email) + token = RefreshToken.for_user(user) + access_token = str(token.access_token) + reset_link = str(settings.CLIENT_URL_ROOT+'/reset-password?token='+access_token) + subject = 'Rest Password' + title = 'Reset Password' + pre_header = 'Reset Password' + pre_content = 'Click the link below to reset your password.' + greeting = f'Hi there,' - elif 'testcase' in e['data_type']: - status = 'Failed' - if e['value'] == 'True': - status = 'Passed' - data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status) - - - exp_list.append(data_type) - - if is_email: - return exp_list - - exp_str = ('\t'+''.join(exp_list)) - return exp_str - - - - - -def create_json_data(data, obj): - json_data = data - item = obj - - for key in json_data: - if 'test_score' == json_data[key]: - json_data[key] = item.score - elif 'seo_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["seo_delta"] - elif 'pwa_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["pwa_delta"] - elif 'crux_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["crux_delta"] - elif 'best_practices_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["best_practices_delta"] - elif 'performance_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["performance_delta"] - elif 'accessibility_delta' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"] - elif 'current_health' == json_data[key]: - json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2) - elif 'health' == json_data[key]: - json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2) - elif 'logs' == json_data[key]: - json_data[key] = len(item.logs) - elif 'current_lighthouse_average' == json_data[key]: - json_data[key] = item.lighthouse_delta["scores"]["current_average"] - elif 'current_average' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["current_average"] - elif 'seo' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["seo"] - elif 'pwa' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["pwa"] - elif 'crux' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["crux"] - elif 'best_practice' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["best_practices"] - elif 'performance' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["performance"] - elif 'accessibility' == json_data[key]: - json_data[key] = item.lighthouse["scores"]["accessibility"] - - elif 'current_yellowlab_average' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["current_average"] - elif 'pageWeight_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"] - elif 'requests_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["requests_delta"] - elif 'domComplexity_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"] - elif 'javascriptComplexity_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["javascriptComplexity_delta"] - elif 'badJavascript_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["badJavascript_delta"] - elif 'jQuery_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["jQuery_delta"] - elif 'cssComplexity_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["cssComplexity_delta"] - elif 'badCSS_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["badCSS_delta"] - elif 'fonts_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["fonts_delta"] - elif 'serverConfig_delta' == json_data[key]: - json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"] - - elif 'yellowlab_average' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["globalScore"] - elif 'pageWeight' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["pageWeight"] - elif 'requests' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["requests"] - elif 'domComplexity' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["domComplexity"] - elif 'javascriptComplexity' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["javascriptComplexity"] - elif 'badJavascript' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["badJavascript"] - elif 'jQuery' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["jQuery"] - elif 'cssComplexity' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["cssComplexity"] - elif 'badCSS' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["badCSS"] - elif 'fonts' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["fonts"] - elif 'serverConfig' == json_data[key]: - json_data[key] = item.yellowlab["scores"]["serverConfig"] + context = { + 'greeting': greeting, + 'title' : title, + 'subject' : subject, + 'email': email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : reset_link, + 'home_page' : settings.CLIENT_URL_ROOT, + 'button_text' : 'Rest my password', + 'content' : '', + 'signature' : '- Cheers!', + } - elif 'avg_image_score' == json_data[key]: - json_data[key] = item.images_delta["average_score"] - elif 'image_scores' == json_data[key]: - json_data[key] = [i["score"] for i in item.images_delta["images"]] + # send email + sendgrid_email(message_obj=context) + data = { + 'success': True + } + + else: + data = { + 'success': False + } + + return data - return json_data +def send_invite_link(member: object=None) -> dict: + """ + Sends an invite email to the passed `Member` + Args: + 'member': obj + + Returns: + 'success': bool + """ + # check if member exists as status "pending" + if Member.objects.filter(email=member.email, status="pending").exists(): -def get_item(object_id): - try: - item = Test.objects.get(id=uuid.UUID(object_id)) - item_type = 'Test' - except: - try: - item = Scan.objects.get(id=uuid.UUID(object_id)) - item_type = 'Scan' - except: - try: - item = Testcase.objects.get(id=uuid.UUID(object_id)) - item_type = 'Testcase' - except: - return {'success': False} + # build email data + link = ( + f'{settings.CLIENT_URL_ROOT}/account/join?team={member.account.id}'+ + f'&code={member.account.code}&member={member.id}&email={member.email}' + ) + subject = 'Cursion Invite' + title = 'Cursion Invite' + pre_header = 'Cursion Invite' + pre_content = ( + f'A user with the email "{member.account.user.username}" invited you to join their '+ + f'Team on Cursion. Now just click the link below to accept the invite!' + ) + greeting = 'Hi there,' - data = { - 'item_type': item_type, - 'item': item, - 'success': True - } + context = { + 'greeting': greeting, + 'title' : title, + 'subject' : subject, + 'email': member.email, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : link, + 'home_page' : settings.CLIENT_URL_ROOT, + 'button_text' : 'Accept Invite', + 'content' : '', + 'signature' : '- Cheers!', + } + + # send email + sendgrid_email(message_obj=context) + data = { + 'success': True + } + + else: + data = { + 'success': False + } + return data +def send_remove_alert(member: object=None) -> dict: + """ + Sends a "removed" email to the passed `Member` and + deletes member from DB -def automation_email(email=None, automation_id=None, object_id=None): - if email and automation_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site - - # getting object - data = get_item(object_id=object_id) - if not data['success']: - return {'success': False} - - item = data['item'] - item_type = data['item_type'] + Args: + 'member': obj + + Returns: + 'success': bool + """ - exp_list = create_exp_str(item=item, automation=automation, is_email=True) + # check if member exists as status "removed" + if Member.objects.filter(email=member.email, status="removed").exists(): - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) - subject = f'Alert for {site.site_url}' - title = f'Alert for {site.site_url}' - pre_header = f'Alert for {site.site_url}' + # build email data + subject = 'Removed From Account' + title = 'Removed From Account' + pre_header = 'Removed From Account' pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n' - ) - content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' + f'A user with the email "{member.account.user.username}" removed you '+ + f'from their Team on Cursion. Please let us know if there\'s been a mistake.' ) - subject = subject + greeting = 'Hi there,' + context = { + 'greeting' : greeting, 'title' : title, - 'subject': subject, + 'subject' : subject, + 'email': member.email, 'pre_header' : pre_header, 'pre_content' : pre_content, - 'exp_list': exp_list, - 'object_url' : object_url, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'View Site Dashboard', - 'content' : content, - 'email': email, + 'object_url' : None, + 'home_page' : settings.CLIENT_URL_ROOT, + 'content' : '', 'signature' : '- Cheers!', } + # send email sendgrid_email(message_obj=context) - # html_message = render_to_string('api/automation_email.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [email], - # html_message = html_message, - # fail_silently = True, - # ) + # delete member obj + member.delete() data = { 'success': True @@ -336,55 +191,294 @@ def automation_email(email=None, automation_id=None, object_id=None): +def create_exp(obj: object=None, alert: object=None) -> dict: + """ + Builds an expression list (exp_list = []) based + on the passed 'obj' and `Alert`. + + Args: + 'obj' : object (Scan, Test, CaseRun, FlowRun), + 'alert' : object + + Returns: + 'exp_list': list, + 'exp_str' : str, + """ + + # seting defaults + exp_list = [] + exp_str = '' + + # loop through alert expressions + for e in alert.expressions: + + # settign defaults + title = None + data = None + + # generate custom data and scores + if 'test_score' in e['data_type']: + title = 'Test Score' + data = str(round(obj.score, 2)) + if 'test_status' in e['data_type']: + status = '❌ FAILED' + if obj.status == 'passed': + status = '✅ PASSED' + title = 'Test Status' + data = status + if 'caserun_status' in e['data_type']: + status = '❌ FAILED' + if e['value'] == 'passed': + status = '✅ PASSED' + title = f'"{obj.title}"' + data = status + if 'flowrun_status' in e['data_type']: + status = '❌ FAILED' + if e['value'] == 'passed': + status = '✅ PASSED' + title = f'"{obj.title}"' + data = status + + # get title and data if None + if title == None: + definition = get_definition(e['data_type']) + if definition: + title = definition['name'] + data = str(eval(definition['value'])) + + # create data string + data_str = f' {title}: {data}\n' + exp_str += data_str + + # add to exp_list + exp_list.append({ + 'title': title, + 'data': data + }) + + # formating return data + data = { + 'exp_list': exp_list, + 'exp_str': exp_str, + } + + return data + + + + +def transpose_data(string: str=None, obj: object=None, secrets: list=[]) -> dict: + """ + Using 'definitions.py' replaces all vairables with definition data. + + Args: + 'string' : str (to be transposed) + 'obj' : object (Scan, Test, CaseRun, Report, Issue), + 'secrets' : list (account secrets) + + Returns: transposed string + """ + + # decryption helper + def decrypt_secret(value): + f = Fernet(settings.SECRETS_KEY) + decoded = f.decrypt(value) + return decoded.decode('utf-8') + + # create secrets_list + secrets_list = [] + for secret in secrets: + secrets_list.append({ + 'key': '{{'+str(secret.name)+'}}', + 'value': decrypt_secret(secret.value) + }) + + # iterate through secrets and replace data + for item in secrets_list: + string = string.replace( + item['key'], + item['value'] + ) + + # iterate through definitions and + # replace {{vairables}} with str(value) first + for item in definitions: + string = string.replace( + ('{{'+str(item['key'])+'}}'), + str(item['value']) + ) + + # iterate through definitions and replace + # str(value) with eval(str(value)) + for item in definitions: + if item['value'] in string: + value = eval(item['value']) + data = value if value is not None else 0 + string = string.replace( + str(item['value']), + str(data) + ) + + # return updated string + return string + + + + +def get_obj(object_id: str=None) -> dict: + """ + Tries to find an object that matches theh passed 'object_id'. + (Scan, Test, CaseRun, FlowRun, Report, Issue) + + Args: + 'object_id': str, + + Returns: + 'obj' : object, + 'obj_type' : str, + 'success' : bool + """ -def automation_report_email(email=None, automation_id=None, object_id=None): - if email and automation_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site + # init obj + obj = None + obj_type = '' + success = False + # check for obj + if not obj: + try: + obj = Test.objects.get(id=uuid.UUID(object_id)) + obj_type = 'Test' + success = True + except: + pass + if not obj: + try: + obj = Scan.objects.get(id=uuid.UUID(object_id)) + obj_type = 'Scan' + success = True + except: + pass + if not obj: try: - item = Report.objects.get(id=uuid.UUID(object_id)) - item_type = 'Report' + obj = CaseRun.objects.get(id=uuid.UUID(object_id)) + obj_type = 'CaseRun' + success = True except: + pass + if not obj: + try: + obj = FlowRun.objects.get(id=uuid.UUID(object_id)) + obj_type = 'FlowRun' + success = True + except: + pass + if not obj: + try: + obj = Report.objects.get(id=uuid.UUID(object_id)) + obj_type = 'Report' + success = True + except: + pass + if not obj: + try: + obj = Issue.objects.get(id=uuid.UUID(object_id)) + obj_type = 'Issue' + success = True + except: + pass + + # format and return data + data = { + 'obj': obj, + 'obj_type': obj_type, + 'success': success + } + + return data + + + + +def alert_email(email: str=None, alert_id: str=None, object_id: str=None) -> dict: + """ + Sends an alert email to the User with + the passed 'email' + + Args: + 'email' : str, + 'alert_id' : str, + 'object_id' : str + + Returns: + 'success': bool + """ + + # check if data is present + if email and alert_id: + + # get alert + alert = Alert.objects.get(id=alert_id) + schedule = alert.schedule + + # getting object + data = get_obj(object_id=object_id) + if not data['success']: return {'success': False} - exp_list = '' - object_url = str(item.path) - subject = f'Report for {site.site_url}' - title = f'Report for {site.site_url}' - pre_header = f'Report for {site.site_url}' + # getting object data + obj = data['obj'] + obj_type = data['obj_type'] + + # clean obj_type + obj_name = obj_type.replace('Run', '') + + # deciding if "page" or "site" scope + if obj_type == 'CaseRun' or obj_type == 'FlowRun': + url = obj.site.site_url + else: + url = obj.page.page_url + + # build dash link + dash_link = f'{settings.CLIENT_URL_ROOT}/schedule' + + # generating expressions from alert + exp_list = create_exp( + obj=obj, + alert=alert + )['exp_list'] + + # build email data + object_url = f'{settings.CLIENT_URL_ROOT}/{obj_type.lower()}/{str(obj.id)}' + subject = f'Alert for {url}' + title = f'Alert for {url}' + pre_header = f'Alert for {url}' pre_content = ( - f'Scanerr just finished creating a {item_type} for {site.site_url}. ' - f'Please click the link below to access and download the report.\n' + f'Cursion just finished running a {obj_name} for {url}. ' + f'Below are the current stats:' ) content = ( - f'This message was triggered by an automation created with Scanerr. ' - f'You can change the automation and schedule in your site\'s dashboard. ' + f'This message was triggered by an alert you created. ' + f'You can change the alert and schedule in your ' + f'dashboard.' ) - subject = subject + context = { 'title' : title, + 'subject': subject, 'pre_header' : pre_header, 'pre_content' : pre_content, 'exp_list': exp_list, 'object_url' : object_url, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'View Report', + 'home_page' : settings.CLIENT_URL_ROOT, + 'button_text' : f'View {obj_name}', 'content' : content, + 'email': email, 'signature' : '- Cheers!', } - html_message = render_to_string('api/automation_email.html', context) - plain_message = strip_tags(html_message) - send_mail( - from_email = os.getenv('EMAIL_HOST_USER'), - subject = subject, - message = plain_message, - recipient_list = [email], - html_message = html_message, - fail_silently = True, - ) + # send email + sendgrid_email(message_obj=context) data = { 'success': True @@ -400,40 +494,69 @@ def automation_report_email(email=None, automation_id=None, object_id=None): -def automation_webhook( - request_type=None, - request_url=None, - request_data=None, - automation_id=None, - object_id=None, - ): - if request_type and automation_id and request_url and request_data and object_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site +def alert_report_email(email: str=None, alert_id: str=None, object_id: str=None) -> dict: + """ + Sends an alert report email to the User with + the passed 'email' - # getting object - data = get_item(object_id=object_id) - if not data['success']: - return {'success': False} + Args: + 'email' : str, + 'alert_id' : str, + 'object_id' : str + + Returns: + 'success': bool + """ - item = data['item'] - item_type = data['item_type'] + # check if data is present + if email and alert_id: - pre_json_data = json.loads(request_data) - json_data = create_json_data(data=pre_json_data, obj=item) + # retrieving user + user = User.objects.get(email=email) + # get alert and deciding if "page" or "site" scope + alert = Alert.objects.get(id=alert_id) + schedule = alert.schedule + + # get `Report` if exists try: - if request_type == 'POST': - response = requests.post(request_url, data=json_data) - elif request_data == 'GET': - response = requests.get(request_url, params=json_data) + report = Report.objects.get(id=uuid.UUID(object_id)) + obj_type = 'Report' + url = report.page.page_url + except: + return {'success': False} - print(response.json()) + # build email data + object_url = str(report.path) + subject = f'Report for {url}' + title = f'Report for {url}' + pre_header = f'Report for {url}' + pre_content = ( + f'Cursion just finished creating a ' + f'Report for {url}. ' + f'Please click the link below to access and download the PDF.' + ) + content = ( + f'\nThis message was triggered by an alert created with Cursion. ' + f'You can change the alert and schedule in your ' + f'dashboard.' + ) - except: - data = {'success': False} + context = { + 'title' : title, + 'subject': subject, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : object_url, + 'home_page' : settings.CLIENT_URL_ROOT, + 'button_text' : 'View Report', + 'content' : content, + 'email': email, + 'signature' : '- Cheers!', + } + # send email + sendgrid_email(message_obj=context) data = { 'success': True } @@ -448,48 +571,73 @@ def automation_webhook( +def alert_phone(phone_number: str=None, alert_id: str=None, object_id: str=None) -> dict: + """ + Sends an SMS alert to the passed 'phone_number' + with the `Alert` data + + Args: + 'phone_number' : str, + 'alert_id' : str, + 'object_id' : str, + + Returns: + 'success': bool + """ -def automation_phone(phone_number=None, automation_id=None, object_id=None): - if phone_number and automation_id and object_id: - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - site = schedule.site + # checking if data is present + if phone_number and alert_id and object_id: + + # getting schedule and alert + alert = Alert.objects.get(id=alert_id) + schedule = alert.schedule + account_id = str(schedule.account.id) # getting object - data = get_item(object_id=object_id) + data = get_obj(object_id=object_id) if not data['success']: return {'success': False} - item = data['item'] - item_type = data['item_type'] + # get obj and type + obj = data['obj'] + obj_type = data['obj_type'] + + # clean obj_type + obj_name = obj_type.replace('Run', '') + + # deciding if "page" or "site" scope + if obj_type == 'CaseRun' or obj_type == 'FlowRun': + url = obj.site.site_url + else: + url = obj.page.page_url + + # build dash link + dash_link = f'{settings.CLIENT_URL_ROOT}/schedule' - exp_str = create_exp_str(item=item, automation=automation) + # build the exp_str + exp_str = create_exp(obj=obj, alert=alert)['exp_str'] - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + # build message data + object_url = f'{settings.CLIENT_URL_ROOT}/{obj_type.lower()}/{obj.id}' pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' + f'Cursion just finished running a {obj_name} for {url}. ' + f'Below are the current stats:\n\n{exp_str}\n' + f'View {obj_name}: {object_url}\n\n' ) content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' + f'This message was triggered by an alert you created. ' + f'You can change the alert and schedule in your dashboard: {dash_link}' ) + body = f'Hi there,\n\n{pre_content}{content}' - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - account_sid = os.environ.get("TWILIO_SID") - auth_token = os.environ.get("TWILIO_AUTH_TOKEN") - client = Client(account_sid, auth_token) - - message = client.messages.create( - to=phone_number, - from_=os.environ.get('TWILIO_NUMBER'), + # send message + data = send_phone( + account_id=account_id, + object_id=object_id, + phone_number=phone_number, body=body ) - - data = { - 'success': True - } + return data else: data = { @@ -501,60 +649,73 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None): -def automation_slack(automation_id=None, object_id=None): - if automation_id and object_id: - automation = Automation.objects.get(id=automation_id) - account = Account.objects.get(user=automation.user) - schedule = automation.schedule - site = schedule.site +def alert_slack(alert_id: str=None, object_id: str=None) -> dict: + """ + Sends a Slack alert with the `Alert` data + + Args: + 'alert_id' : str, + 'object_id' : str, + + Returns: + 'success': bool + """ + + # check if data is present + if alert_id and object_id: + + # getting schedule, account and alert + alert = Alert.objects.get(id=alert_id) + schedule = alert.schedule + account = schedule.account + # getting object - data = get_item(object_id=object_id) + data = get_obj(object_id=object_id) if not data['success']: return {'success': False} - item = data['item'] - item_type = data['item_type'] + # get obj and type + obj = data['obj'] + obj_type = data['obj_type'] + + # deciding if "page" or "site" scope + if obj_type == 'CaseRun' or obj_type == 'FlowRun': + url = obj.site.site_url + else: + url = obj.page.page_url + + # build dash link + dash_link = f'{settings.CLIENT_URL_ROOT}/schedule' - exp_str = create_exp_str(item=item, automation=automation) + # build exp_str + exp_str = create_exp(obj=obj, alert=alert)['exp_str'] - object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id)) + # clean obj_type + obj_name = obj_type.replace('Run', '') + + # build message data + object_url = f'{settings.CLIENT_URL_ROOT}/{obj_type}/{obj.id}' pre_content = ( - f'Scanerr just finished running a {item_type} for {site.site_url}. ' - f'Below are the current stats:\n\n\t{exp_str}\n' + f'Cursion just finished running a `{obj_name}` for {url}. ' + f'Below are the current stats:\n\n```{exp_str}```\n' + f'<{object_url}|*View {obj_name}*>\n\n' ) content = ( - f'This message was triggered by an automation you created. ' - f'You can change the automation and schedule in your site\'s dashboard. ' + f'This message was triggered by an alert you created. ' + f'You can change the alert and schedule in your ' + f'<{dash_link}|dashboard>.' ) + body = f'Hi there,\n\n{pre_content}{content}' - body = f'Hi there,\n\n{pre_content}{content}\n{object_url}' - - token = account.slack['bot_access_token'] - channel = account.slack['slack_channel_id'] - - client = WebClient(token=token) - try: - response = client.chat_postMessage( - channel=channel, - text=(body), - block=[ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": body, - - } - } - ] - ) - except SlackApiError as e: - assert e.response["error"] + # send slack message + data = send_slack( + account_id=account.id, + object_id=object_id, + body=body + ) - data = { - 'success': True - } + return data else: data = { @@ -566,59 +727,85 @@ def automation_slack(automation_id=None, object_id=None): - - - -def sendgrid_email(message_obj): +def sendgrid_email( + account_id: str=None, + object_id: str=None, + message_obj: dict=None + ) -> dict: """ Tries to send an email via the SendGrid API. - Expects the following: - "message_obj": { - 'pre_content': , - 'content': , - 'subject': , - 'title': , - 'pre_header': , - 'button_text': , - 'exp_list': , - 'email': , - 'template': , - 'object_url': , - 'signature': + Expects:{ + 'account_id' : str, + 'object_id' : str, + 'message_obj': dict { + 'plain_text': bool, + 'pre_content': str, + 'content': str, + 'subject': str, + 'title': str, + 'pre_header': str, + 'button_text': str, + 'exp_list': list, + 'email': str, + 'template': str, + 'object_url': str, + 'signature': str, + 'greeting': str, } - - Returns --> data: { - 'message': True - } + + Returns: + 'success': bool, + 'message': str """ - # defining data - pre_content = message_obj.get('pre_content') - content = message_obj.get('content') - subject = message_obj.get('subject') - title = message_obj.get('title') - pre_header = message_obj.get('pre_header') - button_text = message_obj.get('button_text') - email = message_obj.get('email') - exp_list = message_obj.get('exp_list') - object_url = message_obj.get('object_url') - signature = message_obj.get('signature', '- Cheers!') - + plain_text = message_obj.get('plain_text', False) + pre_content = message_obj.get('pre_content', '') + content = message_obj.get('content', '') + subject = message_obj.get('subject', 'Alert from Cursion') + title = message_obj.get('title', '') + pre_header = message_obj.get('pre_header', '') + button_text = message_obj.get('button_text') + emails = message_obj.get('email') + exp_list = message_obj.get('exp_list') + object_url = message_obj.get('object_url') + signature = message_obj.get('signature', '- Cheers!') + greeting = message_obj.get('greeting', 'Hi there,') + + # defaults + success = True + msg = str('') + + if account_id: + # get account & secrets + account = Account.objects.get(id=account_id) + secrets = Secret.objects.filter(account=account) + + # get object + obj = get_obj(object_id)['obj'] + + # cleaning data + content = transpose_data(content, obj, secrets) + subject = transpose_data(subject, obj, secrets) + + # replacing '\n' with
+ content = content.replace('\n', '
') + pre_content = pre_content.replace('\n', '
') # build template data template_data = { - 'title' : title, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : object_url, - 'exp_list': exp_list, - 'home_page' : settings.LANDING_URL_ROOT, - 'button_text' : button_text, - 'content' : content, - 'signature' : signature, - 'subject': subject, + 'greeting' : greeting, + 'title' : title, + 'pre_header' : pre_header, + 'pre_content' : pre_content, + 'object_url' : object_url, + 'exp_list' : exp_list, + 'home_page' : settings.LANDING_URL_ROOT, + 'button_text' : button_text, + 'content' : content, + 'signature' : signature, + 'subject' : subject, } # decide which template to use based on data @@ -628,29 +815,289 @@ def sendgrid_email(message_obj): if exp_list is not None: template = settings.AUTOMATION_TEMPLATE + # loop through passed email addresses + for email in emails.split(','): + + # init SendGrid message + message = Mail( + from_email=From(settings.SENDGRID_EMAIL, 'Cursion'), + to_emails=email.strip(), + ) + + # attach template data and id + if not plain_text: + message.dynamic_template_data = template_data + message.template_id = template + + # building message as plain text + if plain_text: + message.subject = Subject(subject) + message.content = [ + Content( + mime_type="text/html", + content=content + ) + ] + + # send message + try: + sg = SendGridAPIClient(api_key=settings.SENDGRID_API_KEY) + sg.send(message) + msg = f'{msg}, email sent successfully to {email.strip()}' + except Exception as e: + success = False + err = f'error sending email to {email.strip()}' + msg = f'{msg}, {err}' + print(f'{err} | {e}') + + # formatting resposne + data = { + 'success': success, + 'message': msg + } + + return data - # init SendGrid message - message = Mail( - from_email=From('hello@scanerr.io', 'Scanerr'), # prod -> settings.EMAIL_HOST_USER - to_emails=email, - ) + + + +def send_phone( + account_id: str=None, + object_id: str=None, + phone_number: str=None, + body: str=None + ) -> dict: + """ + Using Twilio, sends an SMS with the passed 'body' to the passed + 'phone_number' (single or comma seperated string of phone numbers) + + Args: + 'account_id' : str, + 'object_id' : str, + 'phone_number' : str, + 'body' : str, - # attach template data and id - message.dynamic_template_data = template_data - message.template_id = template + Returns: + 'success': bool, + 'message': str + """ - # send message + if account_id and object_id: + # get account & secrets + account = Account.objects.get(id=account_id) + secrets = Secret.objects.filter(account=account) + + # get object + obj = get_obj(object_id)['obj'] + + # cleaning data + body = transpose_data(body, obj, secrets) + + # defaults + success = True + msg = str('') + + # loop through phone numbers + for number in phone_number.split(','): + + try: + # setup client + account_sid = settings.TWILIO_SID + auth_token = settings.TWILIO_AUTH_TOKEN + client = Client(account_sid, auth_token) + + # clean phone_number + number = number.strip().replace('(', '').replace(')', '').replace('-', '') + number = ''.join(number.split()) + + # send message + client.messages.create( + to=number, + from_=settings.TWILIO_NUMBER, + body=body + ) + msg = f'{msg}, sms sent successfully to {number}' + + except Exception as e: + success = False + err = f'error sending sms to {number}' + msg = f'{msg}, {err}' + print(f'{err} | {e}') + + data = { + 'success': success, + 'message': msg + } + return data + + + + +def send_slack( + account_id: str=None, + object_id: str=None, + body: str=None + ) -> dict: + """ + Using Slack, sends an message with the passed 'body' + to the passed 'account'.channel + + Args: + 'account_id' : str, + 'object_id' : str, + 'body' : str, + + Returns: + 'success': bool, + 'message': str + """ + + if account_id and object_id: + # get account & secrets + account = Account.objects.get(id=account_id) + secrets = Secret.objects.filter(account=account) + + # get object + obj = get_obj(object_id)['obj'] + + # cleaning data + body = transpose_data(body, obj, secrets) + try: - sg = SendGridAPIClient(settings.SENDGRID_API_KEY) - response = sg.send(message) - status = True - except Exception as e: - status = False - print(e.message) + # setup client + token = account.slack['bot_access_token'] + channel = account.slack['slack_channel_id'] + client = WebClient(token=token) + # send message + client.chat_postMessage( + channel=channel, + text=(body), + block=[ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": body, + } + } + ] + ) + success = True + msg = 'slack message sent successfully' + + except SlackApiError as e: + print(e) + success = False + msg = str(e) data = { - 'success': status + 'success': success, + 'message': msg } + return data + + + + +def send_webhook( + account_id: str=None, + object_id: str=None, + request_type: str=None, + url: str=None, + headers: dict=None, + payload: dict=None, + ) -> dict: + """ + Sends a GET or POST request to the passed 'url' + with the passed 'payload' & 'heasders' + + Args: + 'account_id' : str, + 'object_id' : str, + 'request_type' : str, + 'url' : str, + 'headers' : dict, + 'payload' : dict, + + Returns: + 'success': bool, + 'message': str + """ + + # get account & secrets + account = Account.objects.get(id=account_id) + secrets = Secret.objects.filter(account=account) + + # get object + obj = get_obj(object_id)['obj'] + + # normalize values to strings before transposition + raw_headers = headers if isinstance(headers, str) else json.dumps(headers or {}) + raw_payload = payload if isinstance(payload, str) else json.dumps(payload or {}) + raw_url = url if isinstance(url, str) else str(url or '') + # transpose data + cleaned_headers = transpose_data(raw_headers, obj, secrets) + cleaned_payload = transpose_data(raw_payload, obj, secrets) + cleaned_url = transpose_data(raw_url, obj, secrets) + + # defaults for empty inputs + cleaned_headers = cleaned_headers.strip() if cleaned_headers else '{}' + cleaned_payload = cleaned_payload.strip() if cleaned_payload else '{}' + cleaned_url = cleaned_url.strip() if cleaned_url else '' + + # sanitize data + cleaned_headers = re.sub(r'[\x00-\x1f\x7f]', '', cleaned_headers) + cleaned_payload = re.sub(r'[\x00-\x1f\x7f]', '', cleaned_payload) + + # reformatting + cleaned_headers = re.sub(r'(["}])\s*(?=["{])', r'\1,', cleaned_headers) + cleaned_payload = re.sub(r'(["}])\s*(?=["{])', r'\1,', cleaned_payload) + + try: + # building json + json_headers = json.loads(cleaned_headers) if cleaned_headers else {} + if not isinstance(json_headers, dict): + raise ValueError('webhook headers must decode to a JSON object') + + json_payload = {} + if request_type == 'POST': + json_payload = json.loads(cleaned_payload) if cleaned_payload else {} + if not isinstance(json_payload, dict): + raise ValueError('webhook payload must decode to a JSON object') + + # send the request + if request_type == 'POST': + response = requests.post( + url=cleaned_url, + headers=json_headers, + data=json.dumps(json_payload) + ) + + elif request_type == 'GET': + response = requests.get( + url=cleaned_url, + headers=json_headers + ) + + else: + raise ValueError(f'unsupported request_type: {request_type}') + + success = True + try: + msg = str(response.json()) + except Exception: + msg = response.text + + except Exception as e: + success = False + msg = str(e) + + data = { + 'success': success, + 'message': msg + } return data + + diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py new file mode 100644 index 00000000..7563ca76 --- /dev/null +++ b/app/api/utils/archive/caser.py @@ -0,0 +1,899 @@ +# from .driver_p import driver_init as driver_p_init +# from .driver_s import driver_init as driver_init +# from .driver_s import driver_wait, quit_driver +# from .issuer import Issuer +# import time, uuid, json, boto3, os +# from selenium.webdriver.common.by import By +# from selenium.webdriver.common.keys import Keys +# from ..models import * +# from datetime import datetime +# from asgiref.sync import sync_to_async +# from cursion import settings + + + + + + +# class Caser(): +# """ +# Run a `CaseRun` for a specific `Site`. + +# Args: +# 'caserun' : object, +# } + +# - Use `Caser.run_s()` to run with selenium +# - Use `Caser.run_p()` to run with puppeteer + +# Returns: None +# """ + + + + +# def __init__(self, caserun: object=None): +# self.caserun = caserun +# self.site_url = self.caserun.site.site_url +# self.steps = self.caserun.steps +# self.title = self.caserun.case.title +# self.configs = self.caserun.configs +# self.s_keys = { +# '+': Keys.ADD, +# 'Alt': Keys.ALT, +# 'ArrowDown': Keys.ARROW_DOWN, +# 'ArrowLeft': Keys.ARROW_LEFT, +# 'ArrowRight': Keys.ARROW_RIGHT, +# 'ArrowUp': Keys.ARROW_UP, +# 'Backspace': Keys.BACKSPACE, +# 'Control': Keys.CONTROL, +# '.': Keys.DECIMAL, +# 'Delete': Keys.DELETE, +# '/': Keys.DIVIDE, +# 'Enter': Keys.ENTER, +# '=': Keys.EQUALS, +# 'Escape': Keys.ESCAPE, +# 'Meta': Keys.META, +# '*': Keys.MULTIPLY, +# '0': Keys.NUMPAD0, +# '1': Keys.NUMPAD1, +# '2': Keys.NUMPAD2, +# '3': Keys.NUMPAD3, +# '4': Keys.NUMPAD4, +# '5': Keys.NUMPAD5, +# '6': Keys.NUMPAD6, +# '7': Keys.NUMPAD7, +# '8': Keys.NUMPAD8, +# '9': Keys.NUMPAD9, +# ';': Keys.SEMICOLON, +# 'Shift': Keys.SHIFT, +# 'Space': Keys.SPACE, +# '-': Keys.SUBTRACT, +# 'Tab': Keys.TAB +# } + + + + +# @sync_to_async +# def update_caserun( +# self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, +# passed: bool=None, exception: str=None, time_completed: str=None, image: str=None, +# ) -> None: +# # updates Tescase for a puppeteer run (async) +# if start_time != None: +# self.caserun.steps[index][type]['time_created'] = str(start_time) +# if end_time != None: +# self.caserun.steps[index][type]['time_completed'] = str(end_time) +# if passed != None: +# self.caserun.steps[index][type]['passed'] = passed +# if exception != None: +# self.caserun.steps[index][type]['exception'] = str(exception) +# if image != None: +# self.caserun.steps[index][type]['image'] = str(image) +# if time_completed != None: +# self.caserun.time_completed = time_completed +# test_status = True +# for step in self.caserun.steps: +# if step['action']['passed'] == False: +# test_status = False +# if step['assertion']['passed'] == False: +# test_status = False +# self.caserun.passed = test_status + +# self.caserun.save() +# return None + + + + +# def update_caserun_s( +# self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, +# passed: bool=None, exception: str=None, time_completed: str=None, image: str=None, +# ) -> None: +# # updates Tescase for a selenium run (async) +# if start_time != None: +# self.caserun.steps[index][type]['time_created'] = str(start_time) +# if end_time != None: +# self.caserun.steps[index][type]['time_completed'] = str(end_time) +# if passed != None: +# self.caserun.steps[index][type]['passed'] = passed +# if exception != None: +# self.caserun.steps[index][type]['exception'] = str(exception) +# if image != None: +# self.caserun.steps[index][type]['image'] = str(image) +# if time_completed != None: +# self.caserun.time_completed = time_completed +# test_status = True +# for step in self.caserun.steps: +# if step['action']['passed'] == False: +# test_status = False +# if step['assertion']['passed'] == False: +# test_status = False +# self.caserun.passed = test_status + +# self.caserun.save() +# return + + + + +# @sync_to_async +# def format_element(self, element): +# elememt = json.dumps(element).rstrip('"').lstrip('"') +# return str(element) + + + + +# def format_element_s(self, element): +# elememt = json.dumps(element).rstrip('"').lstrip('"') +# return str(element) + + + + +# async def save_screenshot(self, page: object=None) -> str: +# ''' +# Grabs & uploads a screenshot of the `page` +# passed in the params. + +# Returns: `image_url` +# ''' + +# # setup boto3 configurations +# s3 = boto3.client( +# 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), +# aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), +# region_name=str(settings.AWS_S3_REGION_NAME), +# endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) +# ) + +# # setting id for image +# pic_id = uuid.uuid4() + +# # get screenshot +# await page.screenshot({'path': f'{pic_id}.png'}) + +# # seting up paths +# image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') +# remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png' +# root_path = settings.AWS_S3_URL_PATH +# image_url = f'{root_path}/{remote_path}' + +# # upload to s3 +# with open(image, 'rb') as data: +# s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), +# remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} +# ) +# # remove local copy +# os.remove(image) + +# # returning image url +# return image_url + + + + +# def save_screenshot_s(self) -> str: +# ''' +# Grabs & uploads a screenshot of the `page` +# passed in the params. + +# Returns: `image_url` +# ''' + +# # setup boto3 configurations +# s3 = boto3.client( +# 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), +# aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), +# region_name=str(settings.AWS_S3_REGION_NAME), +# endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) +# ) + +# # setting id for image +# pic_id = uuid.uuid4() + +# # get screenshot +# self.driver.save_screenshot(f'{pic_id}.png') + +# # seting up paths +# image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') +# remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png' +# root_path = settings.AWS_S3_URL_PATH +# image_url = f'{root_path}/{remote_path}' + +# # upload to s3 +# with open(image, 'rb') as data: +# s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), +# remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} +# ) +# # remove local copy +# os.remove(image) + +# # returning image url +# return image_url + + + +# @sync_to_async +# def format_exception(self, exception: str) -> str: +# """ +# Cleans the passed `exception` of any +# system refs and unnecessary info + +# Args: +# "exception": str +# } + +# Returns: str +# """ + +# split_e = str(exception).split('Stacktrace:') +# new_exception = split_e[0] + +# return new_exception + + + + +# def format_exception_s(self, exception: str) -> str: +# """ +# Cleans the passed `exception` of any +# system refs and unnecessary info + +# Args: +# "exception": str +# } + +# Returns: str +# """ + +# split_e = str(exception).split('Stacktrace:') +# new_exception = split_e[0] + +# return new_exception + + + + +# def run_s(self) -> None: +# """ +# Runs the self.caserun using selenium as the driver + +# Returns: None +# """ + +# print(f'beginning caserun for {self.site_url} \ +# using case {self.title}') + +# # initate driver +# self.driver = driver_init( +# window_size=self.configs['window_size'], +# device=self.configs['device'] +# ) + +# # setting implict wait_time for driver +# self.driver.implicitly_wait(self.configs['max_wait_time']) + +# i = 0 +# for step in self.steps: +# print(f'-- running step #{i+1} --') + +# # adding catch if nav is not first +# if i == 0 and step['action']['type'] != 'navigate': +# print(f'navigating to {self.site_url} before first step') +# # using selenium, navigate to site root path & wait for page to load +# self.driver.get(f'{self.site_url}') +# time.sleep(int(self.configs['min_wait_time'])) + +# if step['action']['type'] == 'navigate': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'navigating to {self.site_url}{step["action"]["path"]}') +# # using selenium, navigate to requested path & wait for page to load +# driver_wait( +# driver=self.driver, +# interval=int(self.configs.get('interval', 1)), +# min_wait_time=int(self.configs.get('min_wait_time', 3)), +# max_wait_time=int(self.configs.get('max_wait_time', 30)), +# ) +# self.driver.get(f'{self.site_url}{step["action"]["path"]}') +# time.sleep(int(self.configs['min_wait_time'])) +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + + +# if step['action']['type'] == 'scroll': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'scrolling -> {step["action"]["value"]}') + +# # scrolling using plain JavaScript +# self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});') +# time.sleep(int(self.configs.get('min_wait_time', 3))) + +# # get image +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + + +# if step['action']['type'] == 'click': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'clicking element -> {step["action"]["element"]}') +# # using selenium, find and click on the 'element' +# selector = self.format_element_s(step["action"]["element"]) +# element = self.driver.find_element(By.CSS_SELECTOR, selector) + +# # scrolling to element using plain JavaScript +# self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') +# self.driver.execute_script("arguments[0].scrollIntoView();", element) +# self.driver.execute_script("window.scrollBy(0, -100);") +# time.sleep(int(self.configs.get('min_wait_time', 3))) + +# # clicking element +# element.click() +# time.sleep(int(self.configs['min_wait_time'])) +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['action']['type'] == 'change': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'changing element to value -> {step["action"]["value"]}') +# # using selenium, find and change the 'element'.value +# selector = self.format_element_s(step["action"]["element"]) +# element = self.driver.find_element(By.CSS_SELECTOR, selector) + +# # scrolling to element and back down a bit +# self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') +# self.driver.execute_script("arguments[0].scrollIntoView();", element) +# self.driver.execute_script("window.scrollBy(0, -100);") +# time.sleep(int(self.configs.get('min_wait_time', 3))) + +# # changing value of element +# value = step["action"]["value"] +# element.send_keys(value) +# time.sleep(int(self.configs['min_wait_time'])) +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['action']['type'] == 'keyDown': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'keyDown action for key -> {step["action"]["key"]}') +# # getting last known element +# n = (i - 1) +# elm = None +# while True: +# elm = self.steps[n]['action']['element'] +# if elm != None and len(elm) != 0: +# break +# n -= 1 +# selector = self.format_element_s(elm) + +# # using selenium, find elemenmtn and send 'Key' event +# selector = self.format_element_s(step["action"]["element"]) +# element = self.driver.find_element(By.CSS_SELECTOR, selector) + +# # scrolling to element and back down a bit +# self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') +# self.driver.execute_script("arguments[0].scrollIntoView();", element) +# self.driver.execute_script("window.scrollBy(0, -100);") +# time.sleep(int(self.configs.get('min_wait_time', 3))) + +# # using selenium, press the selected key +# element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"])) +# time.sleep(int(self.configs['min_wait_time'])) +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['assertion']['type'] == 'match': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}') +# # using selenium, find elememt and assert if element.text == assertion.text +# selector = self.format_element_s(step["action"]["element"]) +# element = self.driver.find_element(By.CSS_SELECTOR, selector) + +# # scrolling to element and back down a bit +# self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') +# self.driver.execute_script("arguments[0].scrollIntoView();", element) +# self.driver.execute_script("window.scrollBy(0, -100);") +# time.sleep(int(self.configs.get('min_wait_time', 3))) + +# # gettintg elem text +# elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent') +# elementText = elementText.strip() +# print(f'elementText => {elementText}') +# print(f'value => {step["assertion"]["value"]}') + +# # assert text +# assert elementText == step["assertion"]["value"] +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['assertion']['type'] == 'exists': +# exception = None +# passed = True +# self.update_caserun_s( +# index=i, type='assertion', +# start_time=datetime.now() +# ) + +# try: +# print(f'asserting that element -> {step["assertion"]["element"]} exists') +# # using puppeteer, find elememt and assert it exists +# selector = self.format_element_s(step["action"]["element"]) +# element = self.driver.find_element(By.CSS_SELECTOR, selector) + +# # scrolling to element and back down a bit +# self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') +# self.driver.execute_script("arguments[0].scrollIntoView();", element) +# self.driver.execute_script("window.scrollBy(0, -100);") + +# # scrolling to element using plain JavaScript +# self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()') +# element = self.driver.find_element(By.CSS_SELECTOR, selector) +# image = self.save_screenshot_s() + +# except Exception as e: +# image = self.save_screenshot_s() +# exception = self.format_exception_s(e) +# passed = False + +# self.update_caserun_s( +# index=i, type='assertion', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# i += 1 + +# self.update_caserun_s( +# time_completed=datetime.now() +# ) +# quit_driver(driver=self.driver) +# print('-- caserun run complete --') + +# if not self.caserun.passed and self.caserun.configs.get('create_issue'): +# print('generating new Issue...') +# Issuer(caserun=self.caserun).build_issue() + +# return None + + + + +# async def run_p(self) -> None: +# """ +# Runs the self.caserun using pupeteer as the driver + +# Returns: None +# """ + +# print(f'beginning caserun for {self.site_url} \ +# using case {self.title}') + +# # initate driver +# self.driver = await driver_p_init() + +# # init page obj +# self.page = await self.driver.newPage() + +# # setting up page with configs +# sizes = self.configs['window_size'].split(',') +# is_mobile = False +# if self.configs['device'] == 'mobile': +# is_mobile = True + +# self.page_options = { +# 'waitUntil': 'networkidle0', +# 'timeout': int(self.configs['max_wait_time'])*1000 +# } + +# print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s') + +# viewport = { +# 'width': int(sizes[0]), +# 'height': int(sizes[1]), +# 'isMobile': is_mobile, +# } + +# userAgent = ( +# "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ +# (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" +# ) + +# emulate_options = { +# 'viewport': viewport, +# 'userAgent': userAgent +# } + +# if self.configs['device'] == 'mobile': +# await self.page.emulate(emulate_options) +# else: +# await self.page.setViewport(viewport) + + +# i = 0 +# for step in self.steps: +# print(f'-- running step #{i+1} --') + +# # adding catch if nav is not first +# if i == 0 and step['action']['type'] != 'navigate': +# print(f'navigating to {self.site_url} before first step') +# # using puppeteer, navigate to site root path & wait for page to load +# await self.page.goto(f'{self.site_url}', self.page_options) +# time.sleep(int(self.configs['min_wait_time'])) + +# if step['action']['type'] == 'navigate': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'navigating to {self.site_url}{step["action"]["path"]}') +# # using puppeteer, navigate to requested path & wait for page to load +# await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options) +# time.sleep(int(self.configs['min_wait_time'])) +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + + +# await self.update_caserun( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['action']['type'] == 'scroll': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'scrolling -> {step["action"]["value"]}') + +# # scrolling using plain JavaScript +# await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});') +# time.sleep(int(self.configs['min_wait_time'])) + +# # get image +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + +# await self.update_caserun( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['action']['type'] == 'click': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'clicking element -> {step["action"]["element"]}') +# # using puppeteer, find and click on the 'element' +# selector = await self.format_element(step["action"]["element"]) +# await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) +# # scrolling to element using plain JavaScript +# await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') +# element = await self.page.J(selector) +# await element.click() +# time.sleep(int(self.configs['min_wait_time'])) +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + +# await self.update_caserun( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['action']['type'] == 'change': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'changing element to value -> {step["action"]["value"]}') +# # using puppeteer, find and click on the 'element' +# if step["action"]["element"] != (None or ''): +# selector = await self.format_element(step["action"]["element"]) +# await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) +# # scrolling to element using plain JavaScript +# await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') +# element = await self.page.J(selector) +# await element.click(clickCount=3) +# await self.page.keyboard.type(step["action"]["value"]) +# time.sleep(int(self.configs['min_wait_time'])) +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + +# await self.update_caserun( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['action']['type'] == 'keyDown': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='action', +# start_time=datetime.now() +# ) + +# try: +# print(f'keyDown action for key -> {step["action"]["key"]}') +# # using puppeteer, press the selected key +# await self.page.keyboard.press(step['action']['key']) +# time.sleep(int(self.configs['min_wait_time'])) +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + +# await self.update_caserun( +# index=i, type='action', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['assertion']['type'] == 'match': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='assertion', +# start_time=datetime.now() +# ) + +# try: +# print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}') +# # using puppeteer, find elememt and assert if element.text == assertion.text +# selector = await self.format_element(step["assertion"]["element"]) +# await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) +# # scrolling to element using plain JavaScript +# await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()') +# elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent') +# elementText = elementText.strip() +# print(f'elementText => {elementText}') +# print(f'value => {step["assertion"]["value"]}') +# assert elementText == step["assertion"]["value"] +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + +# await self.update_caserun( +# index=i, type='assertion', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# if step['assertion']['type'] == 'exists': +# exception = None +# passed = True +# await self.update_caserun( +# index=i, type='assertion', +# start_time=datetime.now() +# ) + +# try: +# print(f'asserting that element -> {step["assertion"]["element"]} exists') +# # using puppeteer, find elememt and assert it exists +# selector = await self.format_element(step["assertion"]["element"]) +# await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) +# await self.page.J(selector) +# image = await self.save_screenshot(page=self.page) + +# except Exception as e: +# image = await self.save_screenshot(page=self.page) +# exception = await self.format_exception(e) +# passed = False + +# await self.update_caserun( +# index=i, type='assertion', +# end_time=datetime.now(), +# passed=passed, +# exception=exception, +# image=image +# ) + +# i += 1 +# await self.update_caserun( +# time_completed=datetime.now() +# ) +# await self.driver.close() +# print('-- caserun run complete --') + +# if not self.caserun.passed and self.caserun.configs.get('create_issue'): +# print('generating new Issue...') +# Issuer(caserun=self.caserun).build_issue() + +# return None + + + + + + + \ No newline at end of file diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py new file mode 100644 index 00000000..da14dba3 --- /dev/null +++ b/app/api/utils/archive/driver_p.py @@ -0,0 +1,263 @@ +# from pyppeteer import launch +# from cursion import settings +# import time, os, sys, datetime + + + + + + +# async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object: +# """ +# Starts a new puppeteer driver instance + +# Args: +# 'window_size' : str, +# 'wait_time' : int +# } + +# Returns: driver object +# """ + +# # parsing window sizes +# sizes = window_size.split(',') + +# # setting browser options +# options = { +# 'executablePath': os.environ.get('CHROME_BROWSER'), +# 'args': [ +# '--no-sandbox', +# '--disable-dev-shm-usage', +# '--force-device-scale-factor=0.5', +# 'ignore-certificate-errors', +# '--hide-scrollbars', +# f'--window-size={window_size}', +# ], +# 'defaultViewport': { +# 'width': int(sizes[0]), +# 'height': int(sizes[1]), +# }, +# # 'timeout': wait_time * 1000 +# } + +# # launching driver +# driver = await launch( +# options=options, +# headless=True, +# handleSIGINT=False, +# handleSIGTERM=False, +# handleSIGHUP=False +# ) + +# # return driver +# return driver + + + + +# async def interact_with_page(page: object=None) -> object: +# # simulate mouse movement +# # and returns the page object +# await page.mouse.move(0, 0) +# await page.mouse.move(0, 50) +# return page + + + + +# async def wait_for_page(page: object=None, max_wait_time: int=30) -> object: +# """ +# Expects the puppeteer page instance and waits +# for either the page to fully load or the max_wait_time +# to expire before returning. + +# Args: +# 'page' : object, +# 'max_wait_time' : int +# } + +# Returns: page +# """ + +# print(f'waiting for page load or {str(max_wait_time)} seconds') + +# timeout = 0 +# page_state = 'loading' + +# while int(timeout) < int(max_wait_time) and page_state != 'complete': +# page_state = await page.evaluate('document.readyState') +# print(f'document state is {page_state}') +# time.sleep(1) +# timeout += 1 + +# return page + + + + +# async def driver_test() -> None: +# """ +# Spins up a puppeteer driver instance and +# tests to ensure it can access the browser and internet + +# Returns: None +# """ + +# print("Testing puppeteer instalation and integration...") +# message = 'Puppeteer was unable to start\n\n' +# status = 'Failed' + +# # testing puppeteer +# try: +# driver = await driver_init() +# page = await driver.newPage() +# await page.goto('https://google.com', {'waitUntil': 'networkidle0'}) +# await interact_with_page(page) +# title = await page.title() +# assert title == 'Google' +# if title == 'Google': +# status = 'Success' +# message = 'Puppeteer installed and working \N{check mark} \n' + +# # log exception +# except Exception as e: +# print(e) + +# # logging test results +# sys.stdout.write( +# '--- ' + status + ' ---\n'+ message +# ) + +# # quiting driver +# try: +# await driver.close() +# except: +# pass + +# return None + + + + +# async def get_data(url: str=None, configs: dict=None) -> dict: +# """ +# Using the puppeteer driver, navigates to the passed +# 'url' and records the page source and any +# present console errors & warnings + +# Args: +# url : str, +# configs : dict +# } + +# Returns: +# 'html' : str, +# 'logs' : dict, +# } +# """ + +# # initing the driver +# sizes = configs['window_size'].split(',') +# driver = await driver_init(window_size=configs['window_size']) +# page = await driver.newPage() + +# # setting driver configs +# page_options = { +# 'waitUntil': 'networkidle0', +# # 'timeout': configs['max_wait_time']*1000 +# } +# viewport = { +# 'width': int(sizes[0]), +# 'height': int(sizes[1]), +# } +# userAgent = ( +# "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ +# (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36" +# ) +# await page.setViewport(viewport) +# if configs['device'] == 'mobile': +# await page.setUserAgent(userAgent) + +# # defining logs +# logs = [] + +# def record_logs(log): +# # helper method to record console +# # logs in the issues tab +# if log.type == 'error': +# if '.js' in log.text: +# source = 'javascript' +# elif 'http' in log.text: +# source = 'network' +# else: +# source = 'other' +# log_obj = { +# "level": "SEVERE", +# "source": source, +# "message": str(log.text), +# "timestamp": int(datetime.datetime.now().timestamp() * 1000) +# } +# logs.append(log_obj) +# elif log.type == 'warning': +# if '.js' in log.text: +# source = 'javascript' +# elif 'http' in log.text: +# source = 'network' +# else: +# source = 'other' +# log_obj = { +# "level": "WARNING", +# "source": source, +# "message": str(log.text), +# "timestamp": int(datetime.datetime.now().timestamp() * 1000) +# } +# logs.append(log_obj) + +# def record_network(request): +# # helper method to record console +# # network issues in the issues tab +# log_obj = { +# "level": "SEVERE", +# "source": "network", +# "message": f'{request.failure()["errorText"]} {request.url}', +# "timestamp": int(datetime.datetime.now().timestamp() * 1000) +# } +# logs.append(log_obj) + +# def record_error(error): +# # helper method to record console +# # page errors in the issues tab +# err = str(error).split(' at ')[0] +# log_obj = { +# "level": "SEVERE", +# "source": "javascript", +# "message": f'{err}', +# "timestamp": int(datetime.datetime.now().timestamp() * 1000) +# } +# logs.append(log_obj) + +# # getting console logs, warnings, and errors +# page.on('console', lambda log : record_logs(log)) +# page.on('requestfailed', lambda request : record_network(request)) +# page.on('pageerror', lambda error : record_error(error)) + +# # navigate to requested url +# await page.goto(url, page_options) + +# # await page.waitForNavigation(navWaitOpt) +# await wait_for_page(page=page) +# await interact_with_page(page) +# html = await page.content() + +# # quitting driver +# await driver.close() + +# # returning data +# data = { +# 'html': html, +# 'logs': logs, +# } + +# return data + + diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py new file mode 100644 index 00000000..25383597 --- /dev/null +++ b/app/api/utils/archive/imager.py @@ -0,0 +1,797 @@ +# from .driver_s import driver_init, driver_wait, quit_driver +# from .driver_p import driver_init as driver_init_p, wait_for_page +# from ..models import Site, Scan, Test, Mask +# from skimage.metrics import structural_similarity +# from cursion import settings +# from PIL import Image as I, ImageChops, ImageStat +# from datetime import datetime +# from asgiref.sync import sync_to_async +# import time, os, sys, json, uuid, boto3, \ +# statistics, shutil, numpy, cv2 + + + + + + +# class Imager(): +# """ +# High level Image handler used to compare screenshots of +# a website. + +# Also known as VRT or Visual Regression Testing. +# Contains three methods scan_s(), scan_p(), test(). +# The _p appendage denotes using Puppeteer as the webdriver +# and the _s appendage denotes using Selenium as the webdriver: + +# def scan_s(driver=None) -> using selenium +# grabs multiple screenshots of the website +# and uploads them to s3. + +# def scan_p() -> using puppeteer +# grabs multiple screenshots of the website +# and uploads them to s3. + +# def test(test=) -> compares each +# screenshot in the two scans and records +# a score out of 100% + +# """ + + + + +# def __init__(self, scan: object=None, configs: dict=None): + +# # main scan object +# self.scan = scan + +# # main configs object +# self.configs = configs + +# # main image_array for scans +# self.image_array = [] + +# # setup boto3 configurations +# self.s3 = boto3.client( +# 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), +# aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), +# region_name=str(settings.AWS_S3_REGION_NAME), +# endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) +# ) + +# # scripts +# self.pause_video_script = ( +# """ +# document.querySelectorAll('video').forEach(vid => vid.pause()); +# document.querySelectorAll('video').forEach(vid => vid.currentTime=0); +# """ +# ) +# self.set_jquery = ( +# """ +# var jq = document.createElement('script'); +# jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; +# document.getElementsByTagName('head')[0].appendChild(jq); +# """ +# ) +# self.pause_animations_script = ( +# """ +# const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag'); +# const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}'); +# styleElement.appendChild(styleTagCSSes); +# document.head.appendChild(styleElement); +# """ +# ) + + + + +# def check_timeout(self, timeout: int, start_time: str) -> bool: +# """ +# Checks to see if the current time exceedes the alotted timeout. + +# Returns: True if timeout exceeded +# """ +# current = datetime.now() +# diff = current - start_time +# if diff.total_seconds() >= int(timeout): +# print('exceeded timeout') +# return True +# else: +# return False + + + + +# def add_images(self, im1: object, im2: object) -> object: +# """ +# Joins img1 and im2 vertically and saves as "new_img" + +# Returns: new_img +# """ +# im1 = I.open(im1) +# im2 = I.open(im2) +# new_img = I.new('RGB', (im1.width, im1.height + im2.height)) +# new_img.paste(im1, (0, 0)) +# new_img.paste(im2, (0, im1.height)) +# return new_img + + + + +# def save_image(self, pic_id: str, image: object) -> None: +# """ +# Upload image to s3, save info as image_obj, +# add image_obj to image_array, & remove image file + +# Returns: None +# """ +# remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png' +# root_path = settings.AWS_S3_URL_PATH +# image_url = f'{root_path}/{remote_path}' + +# # upload to s3 +# with open(image, 'rb') as data: +# self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), +# remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} +# ) + +# # create image obj and add to list +# img_obj = { +# "index": 0, +# "id": str(pic_id), +# "url": image_url, +# "path": remote_path, +# } +# self.image_array.append(img_obj) + +# print(f'adding {img_obj["url"]} to image_array') + +# # remove local copy +# os.remove(image) + +# return None + + + + +# def scan_s(self, driver: object=None) -> list: +# """ +# Grabs full length screenshots of the website and uploads +# them to s3. + +# Args: +# 'driver': object +# } + +# Returns: self.image_array list +# """ + +# # initialize driver if not passed as param +# driver_present = True +# if not driver: +# driver = driver_init() +# driver_present = False + +# # request page_url +# driver.get(self.scan.page.page_url) + +# # waiting for network requests to resolve +# driver_wait( +# driver=driver, +# interval=int(self.configs.get('interval', 5)), +# min_wait_time=int(self.configs.get('min_wait_time', 10)), +# max_wait_time=int(self.configs.get('max_wait_time', 30)), +# ) + +# # defining browser demesions +# sizes = self.configs.get('window_size', '1920,1080').split(',') + +# # getting full_page_height +# if self.configs.get('auto_height', True): +# full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;") +# sizes = self.configs.get('window_size', '1920,1080').split(',') +# driver.set_window_size(int(sizes[0]), int(full_page_height)) + + +# if self.configs.get('disable_animations') == True: +# # inserting animation pausing script +# try: +# driver.execute_script(self.pause_animations_script) +# except: +# print('cannot pause animations') + +# # inserting video pausing scripts +# try: +# driver.execute_script(self.pause_video_script) +# except: +# print('cannnot pause videos') + +# # mask all listed ids +# if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '': +# ids = self.configs.get('mask_ids').split(',') +# for id in ids: +# try: +# driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") +# print('masked an element') +# except: +# print('cannot find element via id provided') + +# # mask all Global mask ids that are active +# active_masks = Mask.objects.filter(active=True) +# if len(active_masks) != 0: +# for mask in active_masks: +# try: +# driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") +# print('masked an element') +# except: +# print('cannot find element via global mask id provided') + +# # scroll one frame at a time and capture screenshot +# final_img = None +# index = 0 +# last_height = -1 +# bottom = False +# start_time = datetime.now() +# while not bottom: + +# # checking if maxed out time +# if self.check_timeout(self.configs.get('timeout', 300), start_time): +# break + +# # scroll single frame +# if index != 0: +# driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") +# time.sleep(int(self.configs.get('min_wait_time', 10))) + +# # get current position and compare to previous +# new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") +# height_diff = new_height - last_height + +# print(f'new_height => {new_height} | height_diff => {height_diff}') + +# if height_diff > 20: +# last_height = new_height +# pic_id = uuid.uuid4() + +# # waiting for network requests to resolve +# driver_wait( +# driver=driver, +# interval=int(self.configs.get('interval', 5)), +# min_wait_time=int(self.configs.get('min_wait_time', 10)), +# max_wait_time=int(self.configs.get('max_wait_time', 30)), +# ) + +# # get screenshot +# driver.save_screenshot(f'{pic_id}.png') +# image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + +# # resizing image to remove duplicate portions +# img = I.open(image) +# width, height = img.size +# left = 0 +# top = height - (height_diff/2) +# right = width +# _bottom = height +# new_img = img.crop((left, top, right, _bottom)) +# new_img.save(image, quality=100) + +# # adding new image to bottom of existing image (if not index = 0) +# pic_id_2 = uuid.uuid4() +# if index != 0 and final_img is not None: +# self.add_images(final_img, image).save(f'{pic_id_2}.png') +# os.remove(final_img) +# final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') +# else: +# I.open(image).save(f'{pic_id_2}.png') +# final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + +# os.remove(image) +# index += 1 + +# else: +# bottom = True + +# # saving image +# self.save_image(pic_id=pic_id_2, image=final_img) + +# # clean up +# if not driver_present: +# quit_driver(driver) + +# # return images +# return self.image_array + + + + +# async def scan_p(self) -> list: +# """ +# Using Puppeteer, grabs full length screenshots of the website and uploads +# them to s3. + +# Returns: self.image_array list +# """ + +# @sync_to_async +# def get_page(): +# _page = self.scan.page +# return _page + +# # getting Cursion `page` object +# _page = await get_page() + +# # starting up puppeteer driver +# driver = await driver_init_p( +# window_size=self.configs.get('window_size', '1920,1080'), +# wait_time=int(self.configs.get('max_wait_time', 30)) +# ) + +# # initing new puppeteer page +# page = await driver.newPage() + +# # setting configs for driver +# sizes = self.configs.get('window_size', '1920,1080').split(',') +# is_mobile = False +# if self.configs.get('device') == 'mobile': +# is_mobile = True + +# page_options = { +# 'waitUntil': 'networkidle0', +# # 'timeout': int(self.configs.get('max_wait_time', 30))*1000 +# } + +# # requesting page_url to get height of +# await page.goto(_page.page_url, page_options) + +# # waiting for page to load +# await wait_for_page(page=page) + +# # getting full page_height +# page_height = int(sizes[1]) +# if self.configs.get('auto_height', True): +# page_height = await page.evaluate("document.scrollingElement.scrollHeight;") + +# # setting more driver configs +# viewport = { +# 'width': int(sizes[0]), +# 'height': int(page_height), +# 'isMobile': is_mobile, +# } +# userAgent = ( +# "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ +# (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" +# ) +# emulate_options = { +# 'viewport': viewport, +# 'userAgent': userAgent +# } + +# # setting device type +# if self.configs.get('device') == 'mobile': +# await page.emulate(emulate_options) +# else: +# await page.setViewport(viewport) + +# # requesting page_url +# await page.goto(_page.page_url, page_options) + +# # handling anamations +# if self.configs.get('disable_animations') == True: +# try: +# # inserting animation pausing script +# await page.evaluate(self.pause_animations_script) +# except: +# print('cannot pause animations') +# try: +# # pausing videos +# videos = await page.querySelectorAll('video') +# for vid in videos: +# await page.evaluate('(vid) => vid.pause()', vid) +# except Exception as e: +# print(e) + +# # mask all listed ids +# if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '': +# ids = self.configs.get('mask_ids').split(',') +# for id in ids: +# try: +# await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") +# print('masked an element') +# except: +# print('cannot find element via id provided') + + +# # mask all Global mask ids that are active +# @sync_to_async +# def get_active_global_masks(): +# masks = Mask.objects.filter(active=True) +# active_masks = [] +# if len(masks) > 0: +# for mask in masks: +# active_masks.append(mask.id) +# return active_masks + +# active_masks = await get_active_global_masks() + +# for mask in active_masks: +# try: +# await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") +# print('masked an element') +# except: +# print('cannot find element via global mask id provided') + +# @sync_to_async +# def save_image(*args, **kwargs): +# self.save_image(pic_id=pic_id, image=final_img) + +# # scroll one frame at a time and capture screenshot +# final_img = None +# index = 0 +# last_height = -1 +# bottom = False +# start_time = datetime.now() +# while not bottom: + +# # checking if maxed out time +# if self.check_timeout(int(self.configs.get('timeout', 300)), start_time): +# break + +# # scroll single frame +# if index != 0: +# await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") +# time.sleep(int(self.configs.get('min_wait_time', 10))) + +# # get current position and compare to previous +# new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") +# height_diff = new_height - last_height +# if height_diff > 20: +# last_height = new_height +# pic_id = uuid.uuid4() + +# # interact with and wait for page to load +# await page.mouse.move(0, 0) +# await page.mouse.move(0, 100) +# time.sleep(int(self.configs.get('min_wait_time', 10))) +# await wait_for_page(page=page) + +# # get screenshot +# await page.screenshot({'path': f'{pic_id}.png'}) +# image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + +# # resizing image to remove duplicate portions +# img = I.open(image) +# width, height = img.size +# left = 0 +# top = height - (height_diff) +# right = width +# _bottom = height +# new_img = img.crop((left, top, right, _bottom)) +# new_img.save(image, quality=100) + +# # adding new image to bottom of existing image (if not index = 0) +# pic_id_2 = uuid.uuid4() +# if index != 0 and final_img is not None: +# self.add_images(final_img, image).save(f'{pic_id_2}.png') +# os.remove(final_img) +# final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') +# else: +# I.open(image).save(f'{pic_id_2}.png') +# final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + +# os.remove(image) +# index += 1 + +# else: +# bottom = True + +# # saving image +# await save_image(pic_id=pic_id, image=final_img) + +# # cleaning up +# await driver.close() + +# # returning images +# return self.image_array + + + + +# def test(self, test: object, index: int=None) -> dict: +# """ +# Compares each screenshot between the two scans and records +# a score out of 100%. + +# Compairsons used : +# - Structral Similarity Index (ssim) +# - PIL ImageChop Differences, Ratio +# - cv2 ORB Brute-force Matcher, Ratio + +# Args: +# 'test': object, +# 'index': int, +# } + +# Returns: +# 'average_score' : float(0-100), +# 'images' : dict, +# } +# """ + +# # setup temp dirs +# if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')): +# os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}')) + +# # temp root +# temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}') + +# # loop through and download each img in scan and compare it. +# pre_scan_images = test.pre_scan.images +# img_test_results = [] +# scores = [] +# i = 0 + +# if index is not None: +# pre_scan_images = [test.pre_scan.images[index]] +# i = index + +# # catching user error when scan_type +# # did not include 'vrt' +# if pre_scan_images is None: +# images_delta = { +# "average_score": None, +# "images": None, +# } +# return images_delta + + +# for pre_img_obj in pre_scan_images: + +# # getting pre_scan image +# pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') +# with open(pre_img_path, 'wb') as data: +# self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) + +# # getting post_scan image +# try: +# post_img_obj = test.post_scan.images[i] +# except: +# post_img_obj = None + +# if post_img_obj is not None: +# post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') +# with open(post_img_path, 'wb') as data: +# self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) + +# # open images with PIL Image library +# post_img = I.open(post_img_path) +# pre_img = I.open(pre_img_path) + +# # check and reformat image sizes if necessary +# pre_img_w, pre_img_h = pre_img.size +# post_img_w, post_img_h = post_img.size + +# # pre_img is longer +# if pre_img_h > post_img_h: +# print(f'pre_img is larger, adjusting...') +# new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode) +# new_pre_img.save(pre_img_path, quality=100) +# pre_img = I.open(pre_img_path) +# # post_img is longer +# if post_img_h > pre_img_h: +# print(f'post_img is larger, adjusting...') +# new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode) +# new_post_img.save(post_img_path, quality=100) +# post_img = I.open(post_img_path) + + +# # build two new images with differences highlighted +# def highlight_diffs(pre_img_path, post_img_path, index): +# ''' +# Returns: two new images with highlights & float(ssim_score) +# ''' +# # Load the images +# image1 = cv2.imread(pre_img_path) +# image2 = cv2.imread(post_img_path) + +# # Convert the images to grayscale +# gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY) +# gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY) + +# # Compute the SSIM map +# (ssim_score, diff) = structural_similarity(gray1, gray2, full=True) + +# # Highlight the differences +# diff = (diff * 255).astype("uint8") + +# # Threshold the difference map +# _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU) + +# # Find contours of the differences +# contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + +# # Draw rectangles around the differences +# for contour in contours: +# (x, y, w, h) = cv2.boundingRect(contour) +# cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2) +# cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2) + +# # Save the output images +# img_1_id = uuid.uuid4() +# img_2_id = uuid.uuid4() +# cv2.imwrite(temp_root + f"/{img_1_id}.png", image1) +# cv2.imwrite(temp_root + f"/{img_2_id}.png", image2) +# img_objs = save_images(img_1_id, img_2_id, index) + +# data = { +# "img_objs": img_objs, +# "ssim_score": ssim_score +# } + +# return data + + +# # saving old images to new test.id path +# def save_images(pre_img_id, post_img_id, index): +# image_ids = [pre_img_id, post_img_id] +# img_objs = [] +# for img_id in image_ids: +# image = os.path.join(temp_root, f'{img_id}.png') +# remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png' +# root_path = settings.AWS_S3_URL_PATH +# image_url = f'{root_path}/{remote_path}' + +# # upload to s3 +# with open(image, 'rb') as data: +# self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), +# remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} +# ) + +# # building img obj +# obj = { +# "id": str(img_id), +# "url": image_url, +# "path": remote_path, +# "index": index, +# } +# img_objs.append(obj) + +# return img_objs + + +# # test images with PIL +# def pil_score(pre_img, post_img): +# try: +# if (pre_img.mode != post_img.mode) \ +# or (pre_img.size != post_img.size) \ +# or (pre_img.getbands() != post_img.getbands()): +# raise Exception('images are not comparable') + +# # Generate diff image in memory. +# diff_img = ImageChops.difference(pre_img, post_img) + +# # Calculate difference as a ratio. +# stat = ImageStat.Stat(diff_img) +# diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 +# pil_img_score = (100 - diff_ratio) +# # print(f'PIL score -> {pil_img_score}') +# return pil_img_score + +# except Exception as e: +# print(e) + + +# # test with cv2 +# def cv2_score(pre_img, post_img): +# try: +# orb = cv2.ORB_create() + +# # convert to array +# pre_img_array = numpy.array(pre_img) +# post_img_array = numpy.array(post_img) + +# # detect keypoints and descriptors +# kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) +# kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + +# # define the bruteforce matcher object +# bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + +# # perform matches. +# matches = bf.match(desc_a, desc_b) + +# # Look for similar regions with distance < 20. (from 0 to 100) +# similar_regions = [i for i in matches if i.distance < 20] +# if len(matches) == 0: +# cv2_img_score = 100 +# else: +# cv2_img_score = (len(similar_regions) / len(matches)) * 100 +# # print(f'cv2 -> {cv2_img_score}') + +# return cv2_img_score + +# except Exception as e: +# print(e) + + + +# # test images +# try: +# # generating new highlighted images and score via ssim +# ssim_results = highlight_diffs(pre_img_path, post_img_path, i) +# pre_img_diff = ssim_results['img_objs'][0] +# post_img_diff = ssim_results['img_objs'][1] + +# # ssim scoring +# ssim_img_score = ssim_results['ssim_score'] * 100 + +# # pillow scoring +# pil_img_score = pil_score(pre_img, post_img) + +# # pixel perfect scoring +# cv2_img_score = cv2_score(pre_img, post_img) + +# # weighted average +# img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + +# # saving old images to test.id path +# old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i) +# pre_img = old_imgs[0] +# post_img = old_imgs[1] + +# except Exception as e: +# print(e) +# img_score = None +# pre_img = None +# post_img = None +# pre_img_diff = None +# post_img_diff = None + +# # create img test obj and add to array +# img_test_obj = { +# "index": i, +# "pre_img": pre_img, +# "post_img": post_img, +# "pre_img_diff": pre_img_diff, +# "post_img_diff": post_img_diff, +# "score": img_score, +# } + +# img_test_results.append(img_test_obj) +# scores.append(img_score) + +# # remove local copies +# if post_img_obj is not None: +# try: +# os.remove(post_img_path) +# except Exception as e: +# print(e) +# try: +# os.remove(pre_img_path) +# except Exception as e: +# print(e) + +# i += 1 + +# # remove temp dir +# shutil.rmtree(temp_root) + +# # averaging scores and storing in images_delta obj +# try: +# avg_score = statistics.fmean(scores) +# except: +# avg_score = None + +# # formatting response +# images_delta = { +# "average_score": avg_score, +# "images": img_test_results, +# } + +# # returning response +# return images_delta + + + + + diff --git a/app/api/utils/archive/report.py b/app/api/utils/archive/report.py new file mode 100644 index 00000000..24491d83 --- /dev/null +++ b/app/api/utils/archive/report.py @@ -0,0 +1,580 @@ +from ...models import * +from cursion import settings +from reportlab.lib.pagesizes import letter +from reportlab.lib.units import inch +from reportlab.lib.colors import HexColor +from reportlab.pdfgen import canvas +import os, boto3, textwrap, requests + + + + + + +class Report(): + """ + Used for generating web vitals reports for + the associated `Page` & `Scan` + + Args: + 'report': , + 'scan' : + + Use self.generate_report() to create a new report + + Returns: + 'report' : object, + 'success': bool, + 'message': str + """ + + + + + def __init__(self, report: object, scan: object=None): + + # getting report, scan, & page + self.report = report + self.page = self.report.page + self.scan = scan + + # retrieveing latest scan if none + if scan is None: + try: + self.scan = Scan.objects.filter( + page=self.page + ).exclude( + time_completed=None + ).order_by('-time_created')[0] + except Exception as e: + print(e) + self.scan = None + + # building paths & canvas template + if os.path.exists(os.path.join(settings.BASE_DIR, f'reports/')): + self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf') + else: + os.makedirs(f'{settings.BASE_DIR}/reports') + self.local_path = os.path.join(settings.BASE_DIR, f'reports/{self.report.id}.pdf') + + # setting default colors + self.page_index = 0 + self.text_color = self.report.info['text_color'] + self.highlight_color = self.report.info['highlight_color'] + self.background_color = self.report.info['background_color'] + self.c = canvas.Canvas(self.local_path, letter) + self.y = 9 + + # define s3 instance + self.s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + + + + def setup_page(self) -> None: + # sets the defaults for a new page + self.c.setFillColor(HexColor(self.background_color)) + self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + return None + + + + + def end_page(self) -> None: + # adds page number and ends page + self.c.setFont('Helvetica-Bold', 15) + self.c.setFillColor(HexColor(self.text_color)) + self.page_index += 1 + self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.showPage() + return None + + + + + def draw_page_title(self, title: str) -> None: + # adds a title to the given page + self.c.setFont('Helvetica-Bold', 32) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawCentredString(4.25*inch, 10*inch, title) + return None + + + + + def publish_report(self) -> None: + # saves report and uploads to s3 + self.c.save() + remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf' + # uploading package to remote s3 + with open(self.local_path, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + # building and saving report_url + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + return None + + + + + def draw_wrapped_line( + self, + text: str, + length: int, + x_pos: int, + y_pos: int, + y_offset: int + ) -> None: + """ + :param text: the raw text to wrap + :param length: the max number of characters per line + :param x_pos: starting x position + :param y_pos: starting y position + :param y_offset: the amount of space to leave between wrapped lines + """ + # Wraps the passed test at a certain char_length + if len(text) > length: + wraps = textwrap.wrap(text, length, break_long_words=True) + for x in range(len(wraps)): + self.c.drawString(x_pos*inch, y_pos*inch, wraps[x]) + y_pos -= y_offset + y_pos += y_offset # add back offset after last wrapped line + else: + self.c.drawString(x_pos*inch, y_pos*inch, text) + return None + + + + + def cover_page(self) -> None: + """ + Builds the cover page with a title + + Returns: None + """ + + # background and title + self.setup_page() + + # creating dark triangle + p = self.c.beginPath() + p.moveTo(0*inch, 11*inch) + p.lineTo(7*inch, 11*inch) + p.lineTo(2.5*inch, 4.5*inch) + p.lineTo(0*inch, 7*inch) + self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # crating light triangle + p = self.c.beginPath() + p.moveTo(0*inch, 0*inch) + p.lineTo(0*inch, 7*inch) + p.lineTo(7*inch, 0*inch) + self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) + self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) + self.c.drawPath(p, fill=1) + + # date + date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' + self.c.setFont('Helvetica-Bold', 24) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 7.5*inch, date) + + # title + self.c.setFont('Helvetica-Bold', 45) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') + + # page url + font_size = max((30 * (26/len(self.page.page_url))), 16) + self.c.setFont('Helvetica-Bold', font_size) + self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5) + + # cover img + cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") + self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + self.end_page() + return None + + + + + def get_score_data(self, score: float, is_binary: bool=False) -> dict: + """ + Using the passed 'score', decide on + which grade and color to return. + + Args: + 'score' : float, + 'is_binary' : bool + + Returns: dict + """ + + # calc score if binary + score = float(score) + if is_binary: + score = score*100 + + # defining score types + score_types = { + "a": { + "grade": "A", + "color": "#38B43F", + }, + "b": { + "grade": "B", + "color": "#82B436", + }, + "c": { + "grade": "C", + "color": "#ACB43C", + }, + "d": { + "grade": "D", + "color": "#B49836", + }, + "e": { + "grade": "E", + "color": "#B46B34", + }, + "f": { + "grade": "F", + "color": "#B43A29", + }, + + } + + # calculate grade + if score >= 80: + grade = score_types['a'] + elif 80 > score >= 70: + grade = score_types['b'] + elif 70 > score >= 50: + grade = score_types['c'] + elif 50 > score >= 30: + grade = score_types['d'] + elif 30 > score >= 0: + grade = score_types['e'] + else: + grade = score_types['f'] + + # return + return grade + + + + + def get_cat_string(self, cat: str) -> str: + """ + Returns the string coresponding to the passed 'cat' + """ + + if cat == 'fonts': + string = 'Fonts' + elif cat == 'badCSS': + string = 'Bad CSS' + elif cat == 'jQuery': + string = 'jQuery' + elif cat == 'images': + string = 'Images' + elif cat == 'pageWeight': + string = 'Page Weight' + elif cat == 'serverConfig': + string = 'Server Config' + elif cat == 'badJavascript': + string = 'Bad JS' + elif cat == 'cssComplexity': + string = 'CSS Complexity' + elif cat == 'domComplexity': + string = 'DOM Complexity' + elif cat == 'javascriptComplexity': + string = 'JS Complexity' + elif cat == 'seo': + string = 'SEO' + elif cat == 'pwa': + string = 'PWA' + elif cat == 'crux': + string = 'CRUX' + elif cat == 'best_practices' or cat == 'best-practices': + string = 'Best Practices' + elif cat == 'performance': + string = 'Performance' + elif cat == 'accessibility': + string = 'Accessibility' + + return string + + + + + def get_audits(self, uri: str=None) -> dict: + """ + Downloads the JSON file from the passed uri + and return the data as a python dict + """ + if uri: + res = requests.get(uri) + audits = res.json() + return audits + else: + return [] + + + + + def create_data(self, data_type: str) -> None: + """ + Paints the data for the passed 'data_type', + either 'lighthouse' or 'yellowlab'. + + Args: + 'data_type': str + + Returns: None + """ + + # add new page + self.setup_page() + + # decide on which data type + if data_type == 'yellowlab': + data = self.scan.yellowlab + data['audits'] = self.get_audits(data['audits']) + page_title = 'Yellow Lab' + avg_score = 'globalScore' + + if data_type == 'lighthouse': + data = self.scan.lighthouse + data['audits'] = self.get_audits(data['audits']) + page_title = 'Lighthouse' + avg_score = 'average' + + self.draw_page_title(page_title) + if data['scores'][avg_score] is None: + return False + + # measurements + space = .25 + text_space = .05 + begin_y = 8 + log_margin = 3.7 + text_margin = .3 + value_margin = 3 + log_height = .2 + log_width = 4 + grade_tab_width = .07 + + c_count = 0 + logs_count = 0 + for cat in data['audits']: + + # checking if cat is not null + if data['scores'][cat] is not None: + + # creating global score + if c_count == 0: + grade_obj = self.get_score_data((data['scores'][avg_score] or 0)) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + 2*inch, + 8.7*inch, + 1*inch, + 1*inch, + .17*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 30) + self.c.drawCentredString( + 2.5*inch, + 9.05*inch, + grade_obj['grade'] + ) + self.c.setFont('Helvetica', 20) + self.c.drawCentredString( + 5.5*inch, + 8.9*inch, + 'Global Score' + ) + self.c.setFont('Helvetica-Bold', 20) + self.c.drawCentredString( + 5.5*inch, + 9.25*inch, + f'{data["scores"][avg_score]}/100' + ) + + # creating new page at limit --> 20 items + if logs_count >= 20: + self.end_page() + logs_count = 0 + begin_y = 9 + self.setup_page() + self.draw_page_title(f'{page_title} (continued)') + + # creating space btw sections + if c_count > 0 and logs_count != 0: + begin_y = (self.y - .2) + + # creating individual grade cards + grade_obj = self.get_score_data((data['scores'][cat] or 0)) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.roundRect( + .5*inch, + (begin_y - .25)*inch, + .5*inch, + .5*inch, + .12*inch, + stroke=0, + fill=1 + ) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont('Helvetica', 16) + self.c.drawCentredString( + .75*inch, + (begin_y - .07)*inch, + grade_obj['grade'] + ) + + self.c.setFont('Helvetica', 16) + cat_string = self.get_cat_string(cat) + self.c.drawCentredString( + 2.3*inch, + (begin_y - .07)*inch, + cat_string + ) + + p_count = 0 + for policy in data['audits'][cat]: + + if (begin_y - (space * p_count)) < 1: + break + + # setting up keys for dict(s) + if data_type == 'yellowlab': + policy_text = policy["policy"]["label"] + policy_value = policy["value"] + binary = False + if data_type == 'lighthouse': + policy_text = policy["title"] + policy_value = '' + if "displayValue" in policy: + if len(policy["displayValue"]) < 9: + policy_value = policy["displayValue"] + binary = True + + if len(policy_text) < 53: + # creating log box + self.c.setFont('Helvetica', 9) + self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + log_width*inch, log_height*inch, + stroke=0, + fill=1 + ) + + # get grade tab + grade_obj = self.get_score_data((policy['score'] or 0), is_binary=binary) + self.c.setFillColor(HexColor(grade_obj['color'],)) + self.c.rect( + log_margin*inch, + (begin_y - (space * p_count))*inch, + grade_tab_width*inch, + log_height*inch, + stroke=0, + fill=1 + ) + + # inserting data + self.c.setFillColor(HexColor(self.text_color)) + + # text + self.c.drawString( + (log_margin + text_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_text}') + ) + + # value + self.c.drawString( + (value_margin + text_margin + log_margin)*inch, + ((begin_y - (space * p_count)) + text_space)*inch, + (f'{policy_value}') + ) + + p_count += 1 + logs_count += 1 + self.y = (begin_y - (space * p_count)) + + c_count += 1 + + self.end_page() + + return None + + + + + + def generate_report(self) -> dict: + """ + Generates a new Report. + + Returns: + 'report' : object, + 'success': bool, + 'message': str + """ + + # setting defaults + message = 'Scan Page first' + success = False + + # generating if scan is available + if self.scan: + + # add title + self.cover_page() + + # build lighthouse data + if 'lighthouse' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='lighthouse') + + # build yellowlab data + if 'yellowlab' in self.report.type or 'full' in self.report.type: + self.create_data(data_type='yellowlab') + + # save report + self.publish_report() + message = 'Report Generated' + success = True + + # formating response + data = { + 'report' : self.report, + 'success': success, + 'message': message + } + + # returning response + return data + + + + + + + \ No newline at end of file diff --git a/app/api/utils/archive/wordpress_p.py b/app/api/utils/archive/wordpress_p.py new file mode 100644 index 00000000..7897f61f --- /dev/null +++ b/app/api/utils/archive/wordpress_p.py @@ -0,0 +1,570 @@ +# from .driver_p import driver_init +# import time, asyncio, uuid +# from ..models import * +# from datetime import datetime +# from asgiref.sync import sync_to_async + + + + + + +# class Wordpress(): + + +# def __init__( +# self, +# login_url, +# admin_url, +# username, +# password, +# email_address, +# destination_url, +# sftp_address, +# dbname, +# sftp_username, +# sftp_password, +# wait_time, +# process_id +# ): +# # set all global vars +# self.login_url = login_url +# self.username = username +# self.password = password +# self.email_address = email_address +# self.destination_url = destination_url +# self.sftp_address = sftp_address +# self.dbname = dbname +# self.sftp_username = sftp_username +# self.sftp_password = sftp_password +# self.process = Process.objects.get(id=process_id) +# self.native_lang = 'en' + +# if not admin_url.endswith('/'): +# admin_url = admin_url + '/' +# self.admin_url = admin_url + +# if wait_time is None: +# self.wait_time = 30 +# else: +# self.wait_time = wait_time + +# self.navWaitOpt = { +# 'timeout': self.wait_time * 1000, +# 'waitUntil': 'domcontentloaded' +# } + + +# async def login(self): + +# ''' +# Tries to log into a WP site with given credentials. + +# Returns: True / False + +# ''' + +# print('begining login method for ' + self.login_url) + + +# self.driver = await driver_init(wait_time=self.wait_time) + +# # init page obj +# self.page = await self.driver.newPage() +# page_options = { +# 'waitUntil': 'networkidle0', +# 'timeout': self.wait_time * 1000 +# } + +# try: +# await self.page.goto(self.login_url, page_options) +# try: +# await self.page.xpath('//*[@id="user_login"]') +# print('found login form') +# except: +# try: +# jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]') +# await jetpack[0].click() +# await self.page.xpath('//*[@id="user_login"]') +# print('found login form') +# except: +# try: +# login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]") +# await login_link[0].click() +# await self.page.xpath('//*[@id="user_login"]') +# print('found login form') +# except: +# print('unable to locate login form at this path') +# await self.driver.close() +# return False + + +# except: +# print('unable to locate login form at this path') +# await self.driver.close() +# return False + +# user_name_elem = await self.page.xpath('//*[@id="user_login"]') +# await user_name_elem[0].click(clickCount=3) +# await self.page.keyboard.type(self.username) +# time.sleep(1) +# passworword_elem = await self.page.xpath('//*[@id="user_pass"]') +# await passworword_elem[0].click(clickCount=3) +# await self.page.keyboard.type(self.password) +# time.sleep(1) +# await self.page.keyboard.press('Enter') +# await self.page.waitForNavigation(self.navWaitOpt) + + +# try: +# try: +# verify_email = await self.page.xpath('//*[@id="correct-admin-email"]') +# print('need to verify email') +# await verify_email[0].click() +# print('clicked verify') +# except: +# pass + +# print('done with login attempt') + +# try: +# await self.page.xpath('//*[@id="login_error"]') +# print('found login error') +# await self.page.reload() + +# print('trying login again') +# user_name_elem = await self.page.xpath('//*[@id="user_login"]') +# await user_name_elem[0].click(clickCount=3) +# await self.page.keyboard.type(self.username) +# time.sleep(1) +# passworword_elem = await self.page.xpath('//*[@id="user_pass"]') +# await passworword_elem[0].click(clickCount=3) +# await self.page.keyboard.type(self.password) +# time.sleep(1) +# await self.page.keyboard.press('Enter') +# await self.page.waitForNavigation(self.navWaitOpt) + + +# try: +# await self.page.xpath('//*[@id="login_error"]') +# print('found login error again') +# print('counld not login to this site') +# except: +# print('no login errors') + +# except: +# print('no login errors') + +# except: +# print('counld not login to this site') + +# await self.driver.close() +# return False + + +# # removing alerts +# try: +# deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]') +# await deny_btn[0].click() +# print('removed alert') +# except: +# pass +# try: +# # checking if url location is wp-admin +# admin_link = '/wp-admin/' +# current_url = self.page.url +# print('current url -> ' + current_url) +# if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: +# print('inside wp-admin') +# else: +# print('not in wp-admin - navigating there now') +# admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]') +# admin_link = await admin_btn[0].querySelector('a') +# await admin_link[0].click(clickCount=2) +# print('clicked dashboard link') +# await self.page.waitForNavigation(self.navWaitOpt) + + +# except: +# print('could not login') +# await self.driver.close() +# return False + + +# return True + + + + + +# async def begin_lang_check(self): + +# try: +# # navigate to settings +# s_url = 'options-general.php' +# try: +# settings_menu = await self.page.xpath('//*[@id="menu-settings"]') +# await settings_menu[0].click() +# print('clicked settings menu') +# await self.page.waitForNavigation(self.navWaitOpt) +# settings = await self.page.xpath('.//a[@href="'+s_url+'"]') +# await settings[0].click() +# print('clicked settings tab') +# await self.page.waitForNavigation(self.navWaitOpt) + + +# except: +# await self.page.goto(self.page.url + s_url) +# await self.page.waitForNavigation(self.navWaitOpt) + +# # finding and recording current native language +# lang_selector = await self.page.xpath('//*[@id="WPLANG"]') +# optgroup = await lang_selector[0].querySelector('optgroup') +# selected_lang = await optgroup.xpath('.//option[@selected="selected"]') +# default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue() +# default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue() +# print("defalut lang value is " + str(default_lang)) + +# if default_lang != 'en': + +# # selecting english +# await lang_selector[0].select('en_CA') +# print('selected english') + +# # saving settings +# save_btn = await self.page.xpath('//*[@id="submit"]') +# await save_btn[0].click() +# print('saved lang to english') + +# self.native_lang = default_lang_value +# return True + +# else: +# self.native_lang = 'en' + + +# except: +# print('error in changing language') +# return False + + + + + + +# async def end_lang_check(self): + +# if self.native_lang != 'en': + +# try: +# # navigate to settings +# s_url = 'options-general.php' +# try: +# settings_menu = await self.page.xpath('//*[@id="menu-settings"]') +# await settings_menu[0].click() +# print('clicked settings menu') +# await self.page.waitForNavigation(self.navWaitOpt) +# settings = await self.page.xpath('.//a[@href="'+s_url+'"]') +# await settings[0].click() +# print('clicked settings tab') +# await self.page.waitForNavigation(self.navWaitOpt) + +# except: +# await self.page.goto(self.page.url + s_url) +# await self.page.waitForNavigation(self.navWaitOpt) + +# # selecting native lang +# lang_selector = await self.page.xpath('//*[@id="WPLANG"]') +# await lang_selector[0].select(self.native_lang) +# print('selected native_lang') + +# # saving settings +# save_btn = await self.page.xpath('//*[@id="submit"]') +# await save_btn[0].click() +# print('saved native lang') + +# except: +# await self.driver.close() +# return False + +# await self.driver.close() +# return True + + + +# async def install_plugin(self, plugin_name): + +# # setting url for link naving +# plugin_menu_page = 'plugins.php' +# add_plugin_page = 'plugin-install.php' + +# # navigating to plugin page +# try: +# print('trying click method') +# plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]') +# await plugin_menu[0].click() +# await self.page.waitForNavigation(self.navWaitOpt) +# p_url = 'plugins.php' +# plugins = await self.page.xpath('.//a[@href="'+p_url+'"]') +# await plugins[0].click() +# print('clicked plugin menu') +# await self.page.waitForNavigation(self.navWaitOpt) + + +# # looking for dependencies in plugin table +# time.sleep(10) +# form = await self.page.xpath('//*[@id="bulk-action-form"]') +# pluginTable = await form[0].querySelector('tbody') +# tableText = await (await pluginTable.getProperty('textContent')).jsonValue() + +# except: +# print('trying link method for navigation') +# try: +# await self.page.goto(self.admin_link + plugin_menu_page) +# await self.page.waitForNavigation(self.navWaitOpt) + +# time.sleep(10) +# # looking for dependencies in plugin table +# form = await self.page.xpath('//*[@id="bulk-action-form"]') +# pluginTable = await form[0].querySelector('tbody') +# tableText = await (await pluginTable.getProperty('textContent')).jsonValue() +# except: +# print('unable to find plugin table') +# await self.driver.close() +# return False + +# if plugin_name not in tableText: +# try: +# print('plugin not present, preparing to install') + +# time.sleep(2) +# print('navigating to add plugins page') + +# try: +# url = 'plugin-install.php' +# add_plugin = await self.page.xpath('//a[@href="'+url+'"]') +# await add_plugin[0].click(clickCount=2) +# print('clicked add plugin link') +# await self.page.waitForNavigation(self.navWaitOpt) + +# time.sleep(5) +# except: +# await self.page.goto(self.admin_url + add_plugin_page) +# await self.page.waitForNavigation(self.navWaitOpt) + +# time.sleep(5) + + +# # searching for plugin +# search_form = await self.page.xpath('//input[@type="search"]') +# await search_form[0].click(clickCount=3) +# await self.page.keyboard.type(plugin_name) +# time.sleep(1) +# await self.page.keyboard.press('Enter') +# time.sleep(3) + +# ##### Clicking "install" plugin ###### +# install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly +# await install[0].click(clickCount=2) +# print('clicked -install plugin-') +# time.sleep(30) + + +# #### Clicking "activate" plugin ###### +# await self.page.reload() +# print('reloading page') +# try: +# await self.page.waitForNavigation(self.navWaitOpt) +# except: +# pass +# activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly +# await activate[0].click(clickCount=2) +# print('clicked -Activate plugin-') +# time.sleep(30) +# print('Dependencies installed sucessfully') +# return True + +# except: +# print('failed dependency installation') +# await self.driver.close() +# return False + +# else: +# print('plugin already installed') +# return True + + +# @sync_to_async +# def update_process(self, successful=False, info_url=None, time_completed=None, progress=None): +# if info_url is not None: +# self.process.info_url = info_url +# self.process.success = successful +# if time_completed is not None: +# self.process.time_completed = time_completed +# if progress is not None: +# self.process.progress = progress + +# self.process.save() +# return + + + +# async def launch_migration(self): +# ''' +# Launches the migration plugin once Activated. + +# Returns: True / False + +# ''' + +# # setting url for link naving +# migrate_page = 'admin.php?page=cloudways' +# current_url = self.page.url + +# if not current_url.endswith("cloudways"): +# print('navigating to migration page') +# if self.admin_url.endswith('/'): +# await self.page.goto(f'{self.admin_url}{migrate_page}') +# else: +# await self.page.goto(f'{self.admin_url}/{migrate_page}') +# time.sleep(10) + + +# # wait for cloudways email field to become visible +# # entering self.email_address in field +# email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input') +# await email[0].click(clickCount=3) +# await self.page.keyboard.type(self.email_address) +# print('entered cloudways email') + +# # checking T&S checbox +# checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]') +# await checkbox[0].click(clickCount=1) +# print('checked T&S agreement') + +# # clicking submit to launch migration plugin +# m_button = await self.page.xpath('//*[@id="migratesubmit"]') +# await m_button[0].click(clickCount=1) +# print('clicked migrate button') + +# return True + + +# async def run_migration(self): +# ''' +# Enters data on migration page, initiates miration +# and begins updating the associated `Process` with data +# from the page. + +# Returns: True / False + +# ''' + +# # check for page to fully load +# print('waiting 10 sec for new page to load') +# time.sleep(10) +# ## enter all necessary data in each field +# await self.page.waitForNavigation(self.navWaitOpt) + +# # get_element_by_name="address" -> self.destination_url +# destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]') +# await destination_url[0].click(clickCount=3) +# await self.page.keyboard.type(self.destination_url) +# print(f'dest_url as -> {self.destination_url}') +# time.sleep(2) + +# # get_element_by_name="newurl" -> self.sftp_address +# sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]') +# await sftp_address[0].click(clickCount=3) +# await self.page.keyboard.type(self.sftp_address) +# print(f'sftp_address as -> {self.sftp_address}') +# time.sleep(2) + +# # get_element_by_name="appfolder" -> self.dbname +# dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]') +# await dbname[0].click(clickCount=3) +# await self.page.keyboard.type(self.dbname) +# print(f'dbname as -> {self.dbname}') +# time.sleep(2) + +# # get_element_by_name="username" -> self.sftp_username +# sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]') +# await sftp_username[0].click(clickCount=3) +# await self.page.keyboard.type(self.sftp_username) +# print(f'sftp_username as -> {self.sftp_username}') +# time.sleep(2) + +# # get_element_by_name="passwd" -> self.sftp_password +# sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]') +# await sftp_password[0].click(clickCount=3) +# await self.page.keyboard.type(self.sftp_password) +# print(f'sftp_password as -> {self.sftp_password}') +# time.sleep(2) + +# print('entered all creds') + +# # submit data +# await self.page.keyboard.press('Enter') +# print('pressed enter key') + + + +# # update self.process with info_url +# info_url = self.page.url +# await self.update_process(info_url=info_url) + + +# done = False +# done_text = 'Your migration is complete!' +# new_progress = 0 +# print(f'current url -> {self.page.url}') +# while not done: + +# # checking for progres bar +# try: +# raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]') +# new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue() +# new_progress = float(new_progress.split('%')[0]) +# except Exception as e: +# # print(e) +# pass + + +# # update self.process +# await self.update_process(progress=new_progress) + +# # check if new_progress is 100% +# page_content = await self.page.content() +# if new_progress >= 100 or done_text in page_content: +# time_completed = datetime.now() +# await self.update_process(successful=True, time_completed=time_completed, progress=100) +# done = True + +# # checking for process errors +# if 'alert alert-danger' in page_content: +# done = True +# time_completed = datetime.now() +# await self.update_process(time_completed=time_completed) +# print('found an error - ending process') +# return False + +# time.sleep(1) + + +# return True + + + + + + +# async def run_full(self, plugin_name): +# data = await self.login() +# data = await self.begin_lang_check() +# data = await self.install_plugin(plugin_name) +# # data = await self.end_lang_check() +# data = await self.launch_migration() +# data = await self.run_migration() +# await self.driver.close() +# return data + diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py new file mode 100644 index 00000000..85002257 --- /dev/null +++ b/app/api/utils/autocaser.py @@ -0,0 +1,1143 @@ +from .driver import driver_init, driver_wait, quit_driver +from selenium.webdriver.common.by import By +from ..models import Site, Case +from cursion import settings +import time, os, json, uuid, random, boto3 + + + + + + +class AutoCaser(): + """ + Generate new `Cases` for the passed 'site'. + + Args: + 'site' : object, + 'process' : object, + 'start_url' : str, + 'configs' : dict, + 'max_cases' : int, + 'max_layers' : int + } + + Use `AutoCaser.build_cases()` to generate new `Cases` + + Returns: None + """ + + + def __init__( + self, + site : object, + process : object, + start_url : str=None, + configs : dict=settings.CONFIGS, + max_cases : int=4, + max_layers : int=5, + ): + + # main objects & configs + self.site = site + self.process = process + self.start_url = start_url + self.configs = configs + self.max_cases = max_cases + self.max_layers = max_layers + + # high-level elemets array. + # All elememts represent the + # begining of a new Case. + self.elements = [] + self.final_start_elements = [] + + # starting driver + self.driver = driver_init( + browser=self.configs.get('browser'), + window_size=self.configs.get('window_size'), + device=self.configs.get('device'), + ) + + # setting selector script + self.selector_script = ( + """ + const getSelector = (elm) => { + if (elm.tagName === "BODY") return "BODY"; + const names = []; + while (elm.parentElement && elm.tagName !== "BODY") { + if (elm.id) { + names.unshift(`[id='${elm.getAttribute("id")}']`); // "#" + elm.getAttribute("id") + break; + } else { + let c = 1, e = elm; + for (; e.previousElementSibling; e = e.previousElementSibling, c++) ; + names.unshift(elm.tagName + ":nth-child(" + c + ")"); + } + elm = elm.parentElement; + } + return names.join(">"); + } + + return getSelector(arguments[0]) + + """ + ) + + # setting xpath script + self.xpath_script = ( + """ + const getXPath = (elm) => { + const idx = (sib, name) => sib + ? idx(sib.previousElementSibling, name||sib.localName) + (sib.localName == name) + : 1; + const segs = elm => !elm || elm.nodeType !== 1 + ? [''] + : elm.id && document.getElementById(elm.id) === elm + ? [`id("${elm.id}")`] + : [...segs(elm.parentNode), `${elm.localName.toLowerCase()}[${idx(elm)}]`]; + return segs(elm).join('/'); + } + + return getXPath(arguments[0]) + + """ + ) + + # setting selector script + self.visible_script = ( + """ + const isVisible = (elm) => { + try{ + if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){ + return false + } else { + return true + } + }catch{ + return false + } + } + + return isVisible(arguments[0]) + + """ + ) + + # setting defaults for inputs + self.input_types = { + "button": {'test_data': None, 'action': 'click'}, + "checkbox": {'test_data': None, 'action': 'click'}, + "color": {'test_data': '#ff0000', 'action': 'change'}, + "date": {'test_data': '2024-04-23', 'action': 'change'}, + "datetime-local": {'test_data': '2024-04-22T12:49', 'action': 'change'}, + "email": {'test_data': 'jane@example.com', 'action': 'change'}, + "file": {'test_data': None, 'action': None}, + "hidden": {'test_data': None, 'action': None}, + "image": {'test_data': None, 'action': None}, + "month": {'test_data': '2024-04', 'action': 'change'}, + "number": {'test_data': '1', 'action': 'change'}, + "password": {'test_data': 'pass123456!@', 'action': 'change'}, + "radio": {'test_data': None, 'action': 'click'}, + "range": {'test_data': 1, 'action': 'change'}, + "reset": {'test_data': None, 'action': None}, + "search": {'test_data': 'search example', 'action': 'change'}, + "submit": {'test_data': None, 'action': 'click'}, + "tel": {'test_data': '5555555555', 'action': 'change'}, + "text": {'test_data': 'Example Text', 'action': 'change'}, + "time": {'test_data': '12:34', 'action': 'change'}, + "url": {'test_data': 'https://example.com', 'action': 'change'}, + "week": {'test_data': '2024-W15', 'action': 'change'}, + "textarea": {'test_data': 'This is longer example text for testing.', 'action': 'change'}, + "None": {'test_data': None, 'action': None}, + } + + # setting blacklist for input types to ignore + self.blacklist = ['file', 'hidden', 'image', 'reset'] + + # setup boto3 configurations + self.s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + + + + def update_process( + self, + current: int, + total: int, + complete: bool=False, + exception: str=None + ) -> None: + # calculate the current progress of the + # task based on current iteration and total + # iterations expected + final_progress = 90 + progress = 0 + success = False + if complete: + progress = 100 + success = True + if not complete: + progress = float((current/total) * final_progress) + + print(f'updating process --> {progress}%') + + # update Process obj + self.process.progress = progress + self.process.success = success + self.process.save() + + + + + def is_element_visible(self, element: object) -> bool: + try: + resp = self.driver.execute_script(self.visible_script, element) + resp = str(resp).lower() + if resp == 'true': + return True + if resp == 'false': + return False + except Exception as e: + print(f'is_element_visible() Exception -> Stale element reference') + + + + + def get_element_image(self, element: object) -> str: + try: + image = element.screenshot_as_base64 + # sleep for .5 seconds to let image process + time.sleep(.5) + except: + image = None + return image + + + + + def get_url_root(self, url: str) -> str: + protocol = url.split('//')[0] + '//' + root_url = protocol + url.split('//')[1].split('/')[0] + return root_url + + + + + def get_relative_url(self, url: str) -> str: + relative_url = '/' + url.split('//')[1].split('/')[1] + return relative_url + + + + + def get_elem_text(self, selector: str) -> str: + elem_text = self.driver.execute_script(f'return document.querySelector("{selector}").innerText') + elem_text = elem_text.split('\n')[0].strip() + return elem_text + + + + + def get_priority_elements(self, elements: list) -> dict: + priority_words = [ + 'cart', 'checkout', 'add to cart', 'add to the cart', + 'add to basket', 'add to shopping basket', 'add to shopping cart', + 'add to the cart', 'billing', 'address', 'payment', 'purchase now', + 'order now', 'order', 'shop now', 'continue to payment', 'contact', + 'apply', 'submit', 'contact sales', 'contact us' + ] + + priority_elements = [] + non_priority_elements = [] + + # checking each element for prioriry words + for element in elements: + + # get element's innerText + elem_selector = self.driver.execute_script(self.selector_script, element) + elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText') + + # check each priority word against element innerText + for word in priority_words: + if word in elm_text.lower() or elm_text.lower() in word: + priority_elements.append(element) + break + elif element not in non_priority_elements: + non_priority_elements.append(element) + + # if priotity_elements[] is empty + # look for any forms and add them + if len(priority_elements) == 0: + for element in elements: + if element.tag_name == 'form': + # add to priority + priority_elements.append(element) + print('added FORM to priority_elements[]') + + data = { + 'priority_elements': priority_elements, + 'non_priority_elements': non_priority_elements + } + + return data + + + + + def get_current_elements(self) -> list: + # returns a list of interactable + # elements on the current page and + # removes any duplicates before returning + buttons = self.driver.find_elements(By.TAG_NAME, 'button') + links = self.driver.find_elements(By.TAG_NAME, 'a') + forms = self.driver.find_elements(By.TAG_NAME, 'form') + inputs = self.driver.find_elements(By.TAG_NAME, 'input') + textareas = self.driver.find_elements(By.TAG_NAME, 'textarea') + inputs_textareas_buttons = inputs + textareas + buttons + + # get all form inputs, textareas, & buttons + form_elems = [] + for form in forms: + # form inputs + form_inputs = form.find_elements(By.TAG_NAME, 'input') + form_elems += form_inputs + # form textarea + form_textares = form.find_elements(By.TAG_NAME, 'textarea') + form_elems += form_textares + # form buttons + form_buttons = form.find_elements(By.TAG_NAME, 'button') + form_elems += form_buttons + + # then remove duplicates + inputs_textareas_buttons = [elem for elem in inputs_textareas_buttons if elem not in form_elems] + + # shuffle elements in place + random.shuffle(forms) + random.shuffle(inputs_textareas_buttons) + random.shuffle(links) + + current_elements = forms + inputs_textareas_buttons + links + + # return result + return current_elements + + + + + def check_for_duplicates(self, selector: str, elements: list=None) -> bool: + found_duplicate = False + if elements is None: + elements = self.elements + + # checking against all final start elements + for final_start_elem in self.final_start_elements: + if final_start_elem == selector: + found_duplicate = True + return found_duplicate + + for elem in elements: + # check if selector exists already + if elem['selector'] == selector: + found_duplicate = True + break + + # check if sub_elements exists + if elem['elements'] != None: + self.check_for_duplicates(selector=selector, elements=elem['elements']) + + # return result + return found_duplicate + + + + + def get_clean_elements(self, elements: list, check_against: list=None) -> list: + cleaned_elements = [] + current_url = self.driver.current_url + + for elem in elements: + # get slector + elem_selector = self.driver.execute_script(self.selector_script, elem) + + # check local duplicates + if check_against is not None: + if self.check_for_duplicates(selector=elem_selector, elements=check_against): + print(f'found local duplicate => {elem_selector}') + continue + + # check global duplicates + if self.check_for_duplicates(selector=elem_selector): + print(f'found global duplicate => {elem_selector}') + continue + + # check url if + if elem.tag_name == 'a': + # check if action will reload page or site root + elem_link = elem.get_attribute('href') + # check it elem_link is blank + if elem_link is None: + print('elem_link not present') + continue + if current_url == elem_link or elem_link == self.site.site_url or elem_link == '/': + print('elem reloads page') + continue + # check if action will nav to new site + if not elem_link.startswith(self.site.site_url): + print(f'elem links to different site') + continue + + # add to cleaned conditions passed + cleaned_elements.append(elem) + + # return cleaned elements + return cleaned_elements + + + + + def record_new_element(self, elem: object, sub_elements: list) -> dict: + """ + Returns: + 'sub_elements': [], + 'run': bool, + 'added': bool + """ + # setting defaults + run = True + added = False + + # check if element is visible + if not self.is_element_visible(elem): + data = { + 'run': run, + 'added': added, + 'sub_elements': sub_elements + } + return data + + # get sub element info + elem_selector = self.driver.execute_script(self.selector_script, elem) + elem_xpath = self.driver.execute_script(self.xpath_script, elem) + elem_text = self.get_elem_text(selector=elem_selector) + elem_img = self.get_element_image(element=elem) + relative_url = self.get_relative_url(self.driver.current_url) + + # found new element, record, click, & continue + if elem.tag_name == 'a' or elem.tag_name == 'button': + + # record element + sub_elements.append({ + 'selector': elem_selector, + 'xpath': elem_xpath, + 'elem_type': elem.tag_name, + 'elem_text': elem_text, + 'placeholder': None, + 'value': None, + 'type': None, + 'data': None, + 'action': 'click', + 'path': relative_url, + 'img': elem_img, + 'elements': None, + }) + + # click element + try: + elem.click() + except Exception as e: + print('Element not Clickable, removing') + sub_elements.pop() + + # add to layers and ending internal loop + added = True + run = True + + + # found new input or textarea + elif elem.tag_name == 'input' or elem.tag_name == 'textarea': + + # getting element values and type + type = str(elem.get_attribute('type')) + value = elem.get_attribute('value') + if elem.tag_name == 'textarea': + type = 'textarea' + + # record element + sub_elements.append({ + 'selector': elem_selector, + 'xpath': elem_xpath, + 'elem_type': elem.tag_name, + 'elem_text': elem_text, + 'placeholder': elem.get_attribute('placeholder'), + 'value': value, + 'type': type, + 'data': self.input_types[type]['test_data'], + 'action': self.input_types[type]['action'], + 'path': relative_url, + 'img': elem_img, + 'elements': None, + }) + + # add to layers and ending internal loop + added = True + run = True + + + # found new form, record and end run + elif elem.tag_name == 'form': + + # record form into sub_elements list + sub_elements = self.record_forms( + elements=sub_elements, + form=elem + ) + + # add to layers and ending case + added = True + run = False + + data = { + 'sub_elements': sub_elements, + 'run': run, + 'added': added + } + + return data + + + + + def record_forms(self, elements: list, form: object=None) -> list: + + # wait for page to load + driver_wait( + driver=self.driver, + interval=self.configs.get('interval'), + max_wait_time=self.configs.get('max_wait_time'), + min_wait_time=self.configs.get('min_wait_time'), + ) + + # building forms list + if form is None: + # get all forms on the page + forms = self.driver.find_elements(By.TAG_NAME, "form") + else: + # adding single form to que + forms = [form] + + # begin iteration of
gathering + for form in forms: + + # get form selector & xpath + form_selector = self.driver.execute_script(self.selector_script, form) + form_xpath = self.driver.execute_script(self.xpath_script, form) + + print(f'recording form -> {form_selector}') + + # get relative_url + relative_url = self.get_relative_url(self.driver.current_url) + + # getting form text + elem_text = self.get_elem_text(selector=form_selector) + + # get form image + form_img = self.get_element_image(element=form) + + # defining form.elements + sub_elements = [] + + # get all input fields in form + inputs = form.find_elements(By.TAG_NAME, "input") + # iterate through each input + for i in inputs: + + if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i): + # get input data + input_selector = self.driver.execute_script(self.selector_script, i) + input_xpath = self.driver.execute_script(self.xpath_script, i) + placeholder = i.get_attribute('placeholder') + value = i.get_attribute('value') + type = str(i.get_attribute('type')) + img = self.get_element_image(element=i) + + sub_elements.append({ + 'selector': input_selector, + 'xpath': input_xpath, + 'elem_type': i.tag_name, + 'placeholder': placeholder, + 'value': value, + 'type': type, + 'data': self.input_types[type]['test_data'], + 'action': self.input_types[type]['action'], + 'path': relative_url, + 'img': img, + 'elements': None, + }) + + + # get all textarea fields in form + textareas = form.find_elements(By.TAG_NAME, "textarea") + # iterate through each input + for i in textareas: + + if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i): + # get input data + input_selector = self.driver.execute_script(self.selector_script, i) + input_xpath = self.driver.execute_script(self.xpath_script, i) + placeholder = i.get_attribute('placeholder') + type = str(i.get_attribute('type')) + img = self.get_element_image(element=i) + + sub_elements.append({ + 'selector': input_selector, + 'xpath': input_xpath, + 'elem_type': i.tag_name, + 'placeholder': placeholder, + 'value': None, + 'type': type, + 'data': self.input_types['textarea']['test_data'], + 'action': self.input_types['textarea']['action'], + 'path': relative_url, + 'img': img, + 'elements': None, + }) + + + # get all iframes elements in form + iframes = form.find_elements(By.TAG_NAME, "iframe") + # iterate through iframes and save data + for iframe in iframes: + + # get iframe data + iframe_selector = self.driver.execute_script(self.selector_script, iframe) + iframe_xpath = self.driver.execute_script(self.xpath_script, iframe) + iframe_img = self.get_element_image(element=iframe) + + # get all inputs for iframe + iframe_inputs = iframe.find_elements(By.TAG_NAME, "input") + + # iterate through each input + iframe_elements = [] + for i in iframe_inputs: + + if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i): + # get input data + input_selector = self.driver.execute_script(self.selector_script, i) + input_xpath = self.driver.execute_script(self.selector_script, i) + placeholder = i.get_attribute('placeholder') + value = i.get_attribute('value') + type = str(i.get_attribute('type')) + img = self.get_element_image(element=i) + + # save internal iframe data + iframe_elements.append({ + 'selector': input_selector, + 'xpath': input_xpath, + 'elem_type': i.tag_name, + 'placeholder': placeholder, + 'value': value, + 'type': type, + 'data': self.input_types[type]['test_data'], + 'action': self.input_types[type]['action'], + 'path': relative_url, + 'img': img, + 'elements': None, + }) + + # save sub elem data + sub_elements.append({ + 'selector': iframe_selector, + 'xpath': iframe_xpath, + 'elem_type': iframe.tag_name, + 'placeholder': None, + 'value': None, + 'type': None, + 'data': None, + 'action': 'switch', + 'path': relative_url, + 'img': iframe_img, + 'elements': iframe_elements, + }) + + + # get all button elements in form + btns = form.find_elements(By.TAG_NAME, "button") + # iterate through each btn + for btn in btns: + + if self.is_element_visible(btn): + # get button data + btn_selector = self.driver.execute_script(self.selector_script, btn) + btn_xpath = self.driver.execute_script(self.xpath_script, btn) + type = str(btn.get_attribute('type')) + btn_img = self.get_element_image(element=btn) + + sub_elements.append({ + 'selector': btn_selector, + 'xpath': btn_xpath, + 'elem_type': 'button', + 'placeholder': None, + 'value': None, + 'type': type, + 'data': None, + 'elements': None, + 'action': 'click', + 'path': relative_url, + 'img': btn_img, + 'elements': None, + }) + + # save elem data + elements.append({ + 'selector': form_selector, + 'xpath': form_xpath, + 'elem_type': 'form', + 'elem_text': elem_text, + 'value': None, + 'type': None, + 'data': None, + 'action': None, + 'path': relative_url, + 'img': form_img, + 'elements': sub_elements, + }) + + # return elements array + return elements + + + + + def get_elements(self) -> list: + + # get site page + if self.start_url is not None: + self.driver.get(self.start_url) + if self.start_url is None: + self.driver.get(self.site.site_url) + start_page = self.driver.current_url + + # record all forms and sub_elements on page + self.elements = self.record_forms(elements=self.elements) + + # grab all buttons + buttons = self.driver.find_elements(By.TAG_NAME, "button") + + # grab all links + links = self.driver.find_elements(By.TAG_NAME, "a") + + # combine buttons and links + start_elms = buttons + links + + # clean start element + cleaned_start_elems = self.get_clean_elements(start_elms) + + # sorting start_elems + sorted_elements = self.get_priority_elements( + elements=cleaned_start_elems, + ) + priority_elements = sorted_elements['priority_elements'] + non_priority_elements = sorted_elements['non_priority_elements'] + + # ending early if not enough elements to generate with + if len(priority_elements) <= 1 and len(non_priority_elements) <= 1: + return self.elements + + # choosing random priority element + if len(priority_elements) > 0: + choosen = priority_elements[ + random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0 + ] + self.final_start_elements.append( + self.driver.execute_script(self.selector_script, choosen) + ) + + # adding random elements to self.final_start_elements[] + # until max_cases" is reached + iterations = 0 + while (len(self.final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases): + + # random choice + choosen = non_priority_elements[ + random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0 + ] + + # checking if chosen element is visible + if not self.is_element_visible(choosen): + iterations += 1 + continue + + # check if element exists in self.final_start_elements[] + selector = self.driver.execute_script(self.selector_script, choosen) + if selector in self.final_start_elements: + iterations += 1 + continue + + # ensuring link is local to site + if choosen.tag_name == 'a': + link_text = choosen.get_attribute('href') + + # check it elem_link is blank + if link_text is None: + print('link_text not present') + iterations += 1 + continue + + if link_text is not None: + if link_text.startswith(self.get_url_root(start_page)): + self.final_start_elements.append(selector) + + # adding if button + if choosen.tag_name == 'button': + self.final_start_elements.append(selector) + + # forcing loop to quit if not enough cases are created + iterations += 1 + + + # begin elem iteration + iterations = 0 + for selector in self.final_start_elements: + + # ensuring we're at start_page + if self.driver.current_url != start_page: + self.driver.get(start_page) + driver_wait( + driver=self.driver, + interval=self.configs.get('interval'), + max_wait_time=self.configs.get('max_wait_time'), + min_wait_time=self.configs.get('min_wait_time'), + ) + + # getting element by selector + try: + element = self.driver.find_element(By.CSS_SELECTOR, selector) + except Exception as e: + print('Element not Reachable, removing') + self.final_start_elements.remove(selector) + iterations += 1 + continue + + # get element info + element_img = self.get_element_image(element=element) + element_type = element.tag_name + elem_relative_url = self.get_relative_url(self.driver.current_url) + elem_text = self.get_elem_text(selector=selector) + xpath = self.driver.execute_script(self.xpath_script, element) + + print(f'working on this start element -> {selector}') + + # get all current elements and url before action + old_elements = self.get_current_elements() + previous_url = self.driver.current_url + + # perform first action + try: + element.click() + except Exception as e: + print('Element not Clickable, removing') + self.final_start_elements.remove(selector) + continue + + + # begin layering (max_layers) + layers = 0 + run = True + sub_elements = [] + while layers < self.max_layers and run: + + print(f'on layer -> {layers}') + + # driver wait + driver_wait( + driver=self.driver, + interval=self.configs.get('interval'), + max_wait_time=self.configs.get('max_wait_time'), + min_wait_time=self.configs.get('min_wait_time'), + ) + + # check current page + if self.driver.current_url == previous_url: + + # check for new element + new_elements = self.get_current_elements() + + # cleaning new elements + cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements) + + # iterating through each elem + recorded_element = False + for elem in cleaned_elements: + + # record element and increment if necessary + data = self.record_new_element(elem, sub_elements) + run = data['run'] + layers += 1 if data['added'] else 0 + sub_elements = data['sub_elements'] + recorded_element = data['added'] + + # add to layers + if not recorded_element: + layers += 1 + + + # check if page is different but still on site + elif self.driver.current_url != previous_url and \ + self.driver.current_url.startswith(self.get_url_root(previous_url)): + + # get new elements and randomly choose 1 (with priority) + new_elements = self.get_current_elements() + + # cleaning new elements + cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements) + + # sort new elements + sorted_elements = self.get_priority_elements( + elements=cleaned_elements, + ) + priority_elements = sorted_elements['priority_elements'] + non_priority_elements = sorted_elements['non_priority_elements'] + elem = None + + # choosing random priority elememt + if len(priority_elements) > 0: + elem = priority_elements[ + random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0 + ] + print(f'chose priority element | type -> {elem.tag_name}') + + # choosing a random non-priority element + elif len(non_priority_elements) > 0: + elem = non_priority_elements[ + random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0 + ] + print(f'chose non-priority element | type -> {elem.tag_name}') + + # returning early if no elem selected + if not elem: + print('no element was selected') + # add to layers and ending case + layers += 1 + run = False + break + + # record element and increment if necessary + data = self.record_new_element(elem, sub_elements) + run = data['run'] + layers += 1 if data['added'] else 0 + sub_elements = data['sub_elements'] + + # catching all other situations + # naving back to previous_url + if not data['added']: + print('no coditions were met') + # add to layers + layers += 1 + # going back + self.driver.get(previous_url) + + # catching all other situations + # naving back to previous_url + else: + print('no coditions were met') + # add to layers + layers += 1 + # going back + self.driver.get(previous_url) + + + # adding final info to elememt list + self.elements.append({ + 'selector': selector, + 'xpath': xpath, + 'elem_type': element_type, + 'elem_text': elem_text, + 'placeholder': None, + 'value': None, + 'type': None, + 'data': None, + 'action': 'click', + 'path': elem_relative_url, + 'img': element_img, + 'elements': sub_elements, + }) + + # counting for process + iterations += 1 + + # update process + self.update_process(current=iterations, total=len(self.final_start_elements)) + + # quit driver session + quit_driver(self.driver) + + # return elements + return self.elements + + + + + def build_cases(self) -> None: + + # run get_elements + try: + elements = self.get_elements() + except Exception as e: + # catching failures + print(e) + elements = [] + + + # get/decide on value for element + def get_elem_value(element): + if element['value'] == None or len(element['value']) <= 0: + return element['data'] + else: + return element['value'] + + # for each high-level element, + # build a new `Case` and save "steps" + # as .json file uploaded to S3 + for element in elements: + + # defining "steps" + steps = [] + + # adding firt step, which is naving + # to the the starting element's 'path' + steps.append({ + "action":{ + "key": "", + "path": element['path'], + "type": "navigate", + "value": "", + "element": "" + }, + "assertion":{ + "type": "", + "value": "", + "element": { + "selector": "", + "xpath": "", + }, + } + }) + + # adding second step if starting + # element is not a form + if element['elem_type'] != 'form': + steps.append({ + "action":{ + "key": "", + "path": element['path'], + "type": element['action'], + "value": get_elem_value(element), + "element": { + "selector": element['selector'], + "xpath": element['xpath'], + }, + "img": element['img'] + }, + "assertion":{ + "type": "", + "value": "", + "element": { + "selector": "", + "xpath": "", + }, + } + }) + + + # sub_element mapping using recursion + def sub_element_mapping(elements, steps): + if element['elements'] != None: + for elem in elements: + # add step + if elem['action'] is not None: + steps.append({ + "action":{ + "key": "", + "path": elem['path'], + "type": elem['action'], + "value": get_elem_value(elem), + "element": { + "selector": elem['selector'], + "xpath": elem['xpath'], + }, + "img": elem['img'] + }, + "assertion":{ + "type": "", + "value": "", + "element": { + "selector": "", + "xpath": "", + }, + } + }) + + # check if sub_elements exists + if elem['elements'] != None: + sub_element_mapping(elem['elements'], steps) + + # return mapped sub_elements in steps + return steps + + + # add sub_elements to steps + steps = sub_element_mapping(element['elements'], steps) + + # create .json file for steps and upload to s3 + case_id = uuid.uuid4() + + # saving as json file temporarily + with open(f'{case_id}.json', 'w') as fp: + json.dump(steps, fp) + + # seting up paths + steps_file = os.path.join(settings.BASE_DIR, f'{case_id}.json') + remote_path = f'static/cases/{case_id}.json' + root_path = settings.AWS_S3_URL_PATH + steps_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(steps_file, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + + # remove local copy + os.remove(steps_file) + + # save new Case + Case.objects.create( + id = case_id, + site = self.site, + site_url = self.site.site_url, + user = self.site.user, + account = self.site.account, + title = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}', + type = "generated", + processed = False, + steps = { + 'url': steps_url, + 'num_steps': len(steps) + }, + ) + + # update process + self.update_process(current=1, total=1, complete=True) + + + return None + + + + diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py deleted file mode 100644 index ea7be1b9..00000000 --- a/app/api/utils/automations.py +++ /dev/null @@ -1,242 +0,0 @@ -from ..models import * -from .alerts import * -import re, uuid - - - -def automation(automation_id, object_id): - automation = Automation.objects.get(id=automation_id) - schedule = automation.schedule - expressions = automation.expressions - exp_list = [] - actions = automation.actions - act_list = [] - scan = None - test = None - report = None - testcase = None - use_exp = True - - if schedule.task_type == 'scan': - try: - scan = Scan.objects.get(id=object_id) - except: - return False - - elif schedule.task_type == 'test': - try: - test = Test.objects.get(id=object_id) - except: - return False - - elif schedule.task_type == 'report': - try: - report = Report.objects.get(id=object_id) - use_exp = False - except: - return False - elif schedule.task_type == 'testcase': - try: - testcase = Testcase.objects.get(id=object_id) - use_exp = True - except: - return False - else: - return False - - - - - if use_exp: - - for expression in expressions: - - exp = None - data_type = None - - if testcase == None: - value = str(float(re.search(r'\d+', str(expression['value'])).group())) - - if '>=' in expression['operator']: - operator = ' >= ' - else: - operator = ' <= ' - - if 'and' in expression['joiner']: - joiner = ' and ' - elif 'or' in expression['joiner']: - joiner = ' or ' - else: - joiner = '' - - if testcase != None: - operator = ' == ' - joiner = '' - data_type = 'testcase.passed' - value = str(expression['value']) - - - if 'test_score' in expression['data_type']: - data_type = 'float(test.score)' - - # lighthouse test data - elif 'current_lighthouse_average' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["current_average"])' - elif 'seo_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])' - elif 'pwa_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])' - elif 'crux_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])' - elif 'best_practices_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])' - elif 'performance_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])' - elif 'accessibility_delta' in expression['data_type']: - data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])' - # lighthouse scan data - elif 'lighthouse_average' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["average"])' - elif 'seo' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["seo"])' - elif 'pwa' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["pwa"])' - elif 'crux' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["crux"])' - elif 'best_practices' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["best_practices"])' - elif 'performance' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["performance"])' - elif 'accessibility' in expression['data_type']: - data_type = 'float(scan.lighthouse["scores"]["accessibility"])' - - # yellowlab test data - elif 'current_yellowlab_average' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["current_average"])' - elif 'pageWeight_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])' - elif 'requests_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])' - elif 'domComplexity_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])' - elif 'javascriptComplexity_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])' - elif 'badJavascript_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])' - elif 'jQuery_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])' - elif 'cssComplexity_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])' - elif 'badCSS_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])' - elif 'fonts_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])' - elif 'serverConfig_delta' in expression['data_type']: - data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])' - # yellowlab scan data - elif 'yellowlab_average' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["globalScore"])' - elif 'pageWeight' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["pageWeight"])' - elif 'requests' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["requests"])' - elif 'domComplexity' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["domComplexity"])' - elif 'javascriptComplexity' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])' - elif 'badJavascript' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["badJavascript"])' - elif 'jQuery' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["jQuery"])' - elif 'cssComplexity' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])' - elif 'badCSS' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["badCSS"])' - elif 'fonts' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["fonts"])' - elif 'serverConfig' in expression['data_type']: - data_type = 'float(scan.yellowlab["scores"]["serverConfig"])' - - - elif 'logs' in expression['data_type']: - data_type = 'len(scan.logs)' - - elif 'current_health' in expression['data_type']: - data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)' - - elif 'health' in expression['data_type']: - data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)' - - elif 'avg_image_score' in expression['data_type']: - data_type = 'float(test.images_delta["average_score"])' - - elif 'image_scores' in expression['data_type']: - data_type = '[i["score"] for i in test.images_delta["images"]]' - exp = f'{joiner}any(i{operator}{value} for i in {data_type})' - - if exp is None: - exp = f'{joiner}{data_type}{operator}{value}' - - exp_list.append(exp) - - - - for action in actions: - - if 'slack' in action['action_type']: - action_type = f"\n print('sending slack alert')\ - \n automation_slack(automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if 'webhook' in action['action_type']: - action_type = f"\n print('sending webhook alert')\ - \n automation_webhook(request_type='{action['request']}', \ - request_url='{action['url']}', request_data='{action['json']}', \ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if 'email' in action['action_type']: - action_type = f"\n print('sending email alert')\ - \n automation_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if report: - action_type = f"\n print('sending report email')\ - \n automation_report_email(email='{action['email']}',\ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - if 'phone' in action['action_type']: - action_type = f"\n print('sending phone alert')\ - \n automation_phone(phone_number='{action['phone']}', \ - automation_id='{str(automation.id)}', \ - object_id='{str(object_id)}')" - - act = f'{action_type}' - act_list.append(act) - - - exp_string = ' '.join(exp_list) - act_string = ''.join(act_list) - - if not use_exp: - exp_string = '1 == 1' - - automation_logic = f'if {exp_string}:{act_string}' - print(automation_logic) - exec(automation_logic) - - return True - - - - - - - - - - - - \ No newline at end of file diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py index 54f2d814..83f19513 100644 --- a/app/api/utils/caser.py +++ b/app/api/utils/caser.py @@ -1,70 +1,309 @@ -from .driver_p import driver_init -import time, asyncio, uuid, json, boto3, os -from ..models import * -from datetime import datetime from asgiref.sync import sync_to_async -from scanerr import settings +from cryptography.fernet import Fernet +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.action_chains import ActionChains +from .driver import driver_init, driver_wait, quit_driver +from .issuer import Issuer +from .updater import update_flowrun +from .imager import Imager +from ..models import * +from django.utils import timezone +from cursion import settings +import time, uuid, json, boto3, os, requests + class Caser(): + """ + Run a `CaseRun` for a specific `Site` or + gather element info for new `Case`. + Args: + 'caserun' : object, + 'case' : object, + 'process' : object, + 'flowrun_id' : str, + 'node_index' : str, + } - def __init__(self, testcase): - self.testcase = testcase - self.site_url = self.testcase.site.site_url - self.steps = self.testcase.steps - self.case_name = self.testcase.case.name - self.configs = self.testcase.configs + - Use `Caser.run()` to run Case as CaseRun + - Use `Caser.pre_run()` to gather element info for a new Case + Returns: None + """ - @sync_to_async - def update_testcase( - self, index=None, type=None, start_time=None, end_time=None, - passed=None, exception=None, time_completed=None, image=None, + + + def __init__( + self, + case : object=None, + caserun : object=None, + process : object=None, + flowrun_id : str=None, + node_index : str=None, ): + + # primary objects + self.case = case + self.caserun = caserun + self.process = process + + # secondary objects + self.site_url = self.caserun.site.site_url if self.caserun else self.case.site.site_url + self.steps = self.caserun.steps if self.caserun else requests.get(self.case.steps['url']).json() + self.configs = self.caserun.configs if self.caserun else settings.CONFIGS + self.flowrun_id = flowrun_id + self.node_index = node_index + self.account = self.case.account if self.case else self.caserun.account + self.secrets = Secret.objects.filter(account=self.account) + + # init driver + self.driver = driver_init( + browser=self.configs.get('browser', 'chrome'), + window_size=self.configs.get('window_size'), + device=self.configs.get('device') + ) + + # init actions + self.actions = ActionChains(self.driver) + + # Selenium Keys reference + self.s_keys = { + '+': Keys.ADD, + 'Alt': Keys.ALT, + 'ArrowDown': Keys.ARROW_DOWN, + 'ArrowLeft': Keys.ARROW_LEFT, + 'ArrowRight': Keys.ARROW_RIGHT, + 'ArrowUp': Keys.ARROW_UP, + 'Backspace': Keys.BACKSPACE, + 'Control': Keys.CONTROL, + '.': Keys.DECIMAL, + 'Delete': Keys.DELETE, + '/': Keys.DIVIDE, + 'Enter': Keys.ENTER, + '=': Keys.EQUALS, + 'Escape': Keys.ESCAPE, + 'Meta': Keys.META, + '*': Keys.MULTIPLY, + '0': Keys.NUMPAD0, + '1': Keys.NUMPAD1, + '2': Keys.NUMPAD2, + '3': Keys.NUMPAD3, + '4': Keys.NUMPAD4, + '5': Keys.NUMPAD5, + '6': Keys.NUMPAD6, + '7': Keys.NUMPAD7, + '8': Keys.NUMPAD8, + '9': Keys.NUMPAD9, + ';': Keys.SEMICOLON, + 'Shift': Keys.SHIFT, + 'Space': Keys.SPACE, + '-': Keys.SUBTRACT, + 'Tab': Keys.TAB + } + + # common scripts + self.scroll_to_center = ( + """ + const scrollToCenter = (elem) => { +         const rect = elem.getBoundingClientRect(); +         const absoluteElementTop = rect.top + window.pageYOffset; +         const middle = absoluteElementTop - (window.innerHeight / 2) + (rect.height / 2); +         window.scrollTo({top: middle, behavior: 'instant'}); + setTimeout(function() {return null}, 300); + } + return scrollToCenter(arguments[0]) + """ + ) + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': ( + f'starting up driver for case run using {self.configs.get('browser', 'chrome')}' + ), + 'object_id': str(self.caserun.id) + }) + + + + + def transpose_data(self, string: str=None) -> str: + """ + Using replaces all vairables in string with + account `Secrets`. + + Args: + 'string' : str (to be transposed) + } + + Returns: transposed string + """ + + # decryption helper + def decrypt_secret(value): + f = Fernet(settings.SECRETS_KEY) + decoded = f.decrypt(value) + return decoded.decode('utf-8') + + # create secrets_list + secrets_list = [] + for secret in self.secrets: + secrets_list.append({ + 'key': '{{'+str(secret.name)+'}}', + 'value': decrypt_secret(secret.value) + }) + + # iterate through secrets and replace data + for item in secrets_list: + string = string.replace( + item['key'], + item['value'] + ) + + # return transposed str + return string + + + + + def update_caserun( + self, + index: str=None, + type: str=None, + start_time: str=None, + end_time: str=None, + status: str=None, + exception: str=None, + time_completed: str=None, + image: str=None, + ) -> None: + + # updates Tescase for a selenium run (async) if start_time != None: - self.testcase.steps[index][type]['time_created'] = str(start_time) + self.caserun.steps[index][type]['time_created'] = str(start_time) if end_time != None: - self.testcase.steps[index][type]['time_completed'] = str(end_time) - if passed != None: - self.testcase.steps[index][type]['passed'] = passed + self.caserun.steps[index][type]['time_completed'] = str(end_time) + if status != None: + self.caserun.steps[index][type]['status'] = status if exception != None: - self.testcase.steps[index][type]['exception'] = str(exception) + self.caserun.steps[index][type]['exception'] = str(exception) if image != None: - self.testcase.steps[index][type]['image'] = str(image) + self.caserun.steps[index][type]['image'] = str(image) if time_completed != None: - self.testcase.time_completed = time_completed - test_status = True - for step in self.testcase.steps: - if step['action']['passed'] == False: - test_status = False - if step['assertion']['passed'] == False: - test_status = False - self.testcase.passed = test_status + self.caserun.time_completed = time_completed + run_status = 'passed' + for step in self.caserun.steps: + if step['action']['status'] == 'failed': + run_status = 'failed' + if step['assertion']['status'] == 'failed': + run_status = 'failed' + self.caserun.status = run_status - self.testcase.save() - return + # save caserun + self.caserun.save() - @sync_to_async - def format_element(self, element): - elememt = json.dumps(element).rstrip('"').lstrip('"') - return element + # compare image if image was passed + if image is not None: + self.compare_images(index=index, type=type) + return None - async def save_screenshot(self, page): - ''' - Grabs & uploads a screenshot of the `page` - passed in the params. - Returns -> `image_url` + def compare_images(self, index: int=None, type: str=None) -> None: + """ + Using Imager.caserun_vrt compare the step.screeshot + to the Case baseline. + + Args: + 'index' : int, step index + 'type' : str, 'action' or 'assertion' + } + + Returns: None + """ + # run Imager + image_delta_obj = Imager(caserun=self.caserun).caserun_vrt(step=index, type=type) + + # update caserun + self.caserun.steps[index][type]['image_delta'] = image_delta_obj + self.caserun.save() + + return None + + + + + def update_process( + self, + current: int, + total: int, + complete: bool=False, + ) -> None: + """ + Calculates the current progress of the + task based on current step and total + number of steps expected - then updates self.process + with the info. + + Args: + current : int, + total : int, + complete : bool=False, + } + + Returns: None + """ + + final_progress = 90 + progress = 0 + success = False + if complete: + progress = 100 + success = True + if not complete: + progress = float((current/total) * final_progress) + + print(f'updating process --> {progress}%') - ''' + # update Process obj + self.process.progress = progress + self.process.success = success + self.process.save() + + + + + def format_element(self, element: object) -> str: + elememt = json.dumps(element).rstrip('"').lstrip('"') + return str(element) + + + + + def save_screenshot(self, run_type: str=None) -> str: + """ + Grabs & uploads a screenshot of the active `page` + self.driver is working on. + + Args: + run_type: str, 'run' or 'pre_run' + } + + Returns: `image_url` + """ + + # default + image_url = None # setup boto3 configurations s3 = boto3.client( @@ -76,282 +315,1108 @@ async def save_screenshot(self, page): # setting id for image pic_id = uuid.uuid4() + + # catch any timeout/detachment errors + try: + + # get screenshot + self.driver.save_screenshot(f'{pic_id}.png') + + # seting up paths + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + if run_type == 'run': + remote_path = f'static/caseruns/{self.caserun.id}/{pic_id}.png' + if run_type == 'pre_run': + remote_path = f'static/case/{self.case.id}/{pic_id}.png' + + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + # remove local copy + os.remove(image) - # get screenshot - await page.screenshot({'path': f'{pic_id}.png'}) + except Exception as e: + print(e) + + # returning image url + return image_url + + + + + def save_case_steps(self, steps: dict, case_id: str) -> dict: + """ + Helper function that uploads the "steps" data to + s3 bucket + + Args: + 'steps' : dict, + 'case_id' : str + } + + Returns: + 'num_steps' : int, + 'url' : str + } + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + # saving as json file temporarily + steps_id = uuid.uuid4() + with open(f'{steps_id}.json', 'w') as fp: + json.dump(steps, fp) + # seting up paths - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png' + steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json') + remote_path = f'static/cases/{case_id}/{steps_id}.json' root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - + steps_url = f'{root_path}/{remote_path}' + # upload to s3 - with open(image, 'rb') as data: + with open(steps_file, 'rb') as data: s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + remote_path, ExtraArgs={ + 'ACL': 'public-read', + 'ContentType': 'application/json', + 'CacheControl': 'max-age=0' + } ) + # remove local copy - os.remove(image) + os.remove(steps_file) - # returning image url - return image_url + # format data + data = { + 'num_steps': len(steps), + 'url': steps_url + } - - - - async def run(self): + # return response + return data - print(f'beginging testcase for {self.site_url} \ - using case {self.case_name}') - - # initate driver - self.driver = await driver_init() - - # init page obj - self.page = await self.driver.newPage() - - # setting up page with configs - sizes = self.configs['window_size'].split(',') - is_mobile = False - if self.configs['device'] == 'mobile': - is_mobile = True - - self.page_options = { - 'waitUntil': 'networkidle0', - 'timeout': int(self.configs['max_wait_time'])*1000 + + + def get_element(self, selector: str=None, xpath: str=None) -> object: + """ + Tries to get element by selector first and + then by xpath. If both fail, then return + None for "element" and True for "failed". + + Args: + "selector": str, + "xpath": str, + } + + Returns: + 'element': object | None, + 'failed': bool } + """ + + # defaults + failed = True + element = None - print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s') + # try selector first + if selector: + try: + element = self.driver.find_element(By.CSS_SELECTOR, selector) + failed = False + except: + pass + # try xpath as backup + if xpath: + try: + element = self.driver.find_element(By.XPATH, xpath) + failed = False + except: + pass - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, + # return data + data = { + 'element': element, + 'failed': failed } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent + return data + + + + + def format_exception(self, exception: str) -> str: + """ + Cleans the passed `exception` of any + system refs and unnecessary info + + Args: + "exception": str } - if self.configs['device'] == 'mobile': - await self.page.emulate(emulate_options) - else: - await self.page.setViewport(viewport) + Returns: str + """ + + split_e = str(exception).split('Stacktrace:') + new_exception = split_e[0] + + return new_exception + + + + + def get_element_image(self, element: object) -> str: + """ + Grabs a screenshot of the passed "element" + and returns image data as base64 str. + + Args: + "element": object (REQUIRED) + } + + Returns: str (base64 encoded) + """ + + try: + image = element.screenshot_as_base64 + # sleep for .5 seconds to let image process + time.sleep(.5) + except: + image = None + return image + + + + + def run(self) -> None: + """ + Runs the self.caserun using selenium as the driver + + Returns: None + """ + + msg = f'starting case run for {self.site_url} using case "{self.caserun.title}" | run_id: {str(self.caserun.id)}' + print(msg) + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg, + 'objects': [{ + 'parent': str(self.caserun.site.id), + 'id': str(self.caserun.id), + 'status': 'working' + }] + }) + + # setting implict wait_time for driver + self.driver.implicitly_wait(self.configs['max_wait_time']) i = 0 for step in self.steps: - print(f'-- running step #{i+1} --') - # print(f'step contents: {step}') + msg = f'running step #{i+1} | run_id: {str(self.caserun.id)}' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + # adding catch if nav is not first + if i == 0 and step['action']['type'] != 'navigate': + print(f'navigating to {self.site_url} before first step') + # using selenium, navigate to site root path & wait for page to load + self.driver.get(f'{self.site_url}') + time.sleep(int(self.configs['min_wait_time'])) + if step['action']['type'] == 'navigate': exception = None - passed = True - image = None - await self.update_testcase( + status = 'passed' + self.update_caserun( index=i, type='action', - start_time=datetime.now() + start_time=timezone.now() ) try: - print(f'navigating to {self.site_url}{step["action"]["path"]}') - # using puppeteer, navigate to requested path & wait for page to load - await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options) - time.sleep(int(self.configs['min_wait_time'])) + msg = f'navigating to {self.site_url}{step["action"]["path"]} | run_id: {str(self.caserun.id)}' + print(msg) + + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + # using selenium, navigate to requested path & wait for page to load + driver_wait( + driver=self.driver, + interval=int(self.configs.get('interval', 1)), + min_wait_time=int(self.configs.get('min_wait_time', 3)), + max_wait_time=int(self.configs.get('max_wait_time', 30)), + ) + self.driver.get(f'{self.site_url}{step["action"]["path"]}') + time.sleep(int(self.configs.get('min_wait_time', 3))) + image = self.save_screenshot(run_type='run') except Exception as e: - image = await self.save_screenshot(page=self.page) - exception = e - passed = False + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + msg = exception + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) + + # update caserun + self.update_caserun( + index=i, type='action', + end_time=timezone.now(), + status=status, + exception=exception, + image=image + ) + + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + + + if step['action']['type'] == 'scroll': + exception = None + status = 'passed' + self.update_caserun( + index=i, type='action', + start_time=timezone.now() + ) + + try: + msg = f'scrolling ({step["action"]["value"]}) | run_id: {str(self.caserun.id)}' + print(msg) + + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message':msg + }) + + # scrolling using plain JavaScript + self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});') + time.sleep(int(self.configs.get('min_wait_time', 3))) + # get image + image = self.save_screenshot(run_type='run') - await self.update_testcase( + except Exception as e: + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) + + # update caserun + self.update_caserun( index=i, type='action', - end_time=datetime.now(), - passed=passed, + end_time=timezone.now(), + status=status, exception=exception, image=image ) - + + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + + + if step['action']['type'] == 'mouseover': + exception = None + status = 'passed' + self.update_caserun( + index=i, type='action', + start_time=timezone.now() + ) + + try: + msg = f'mouseover element "{step["action"]["element"]["selector"]}" | run_id: {str(self.caserun.id)}' + print(msg) + + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message':msg + }) + + # using selenium, find and moving mouse to the 'element' + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain JavaScript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # moving mouse to element + self.actions.move_to_element(element).perform() + time.sleep(int(self.configs.get('min_wait_time', 3))) + image = self.save_screenshot(run_type='run') + + except Exception as e: + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) + + # update caserun + self.update_caserun( + index=i, type='action', + end_time=timezone.now(), + status=status, + exception=exception, + image=image + ) + + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break if step['action']['type'] == 'click': exception = None - passed = True - image = None - await self.update_testcase( + status = 'passed' + self.update_caserun( index=i, type='action', - start_time=datetime.now() + start_time=timezone.now() ) try: - print(f'clicking element -> {step["action"]["element"]}') - # using puppeteer, find and click on the 'element' - selector = await self.format_element(step["action"]["element"]) - await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) + msg = f'clicking element "{step["action"]["element"]["selector"]}" | run_id: {str(self.caserun.id)}' + print(msg) + + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message':msg + }) + + # using selenium, find and click on the 'element' + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + # scrolling to element using plain JavaScript - await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') - element = await self.page.J(selector) - await element.click() - time.sleep(int(self.configs['min_wait_time'])) + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # clicking element + element.click() + time.sleep(int(self.configs.get('min_wait_time', 3))) + image = self.save_screenshot(run_type='run') except Exception as e: - image = await self.save_screenshot(page=self.page) - exception = e - passed = False + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) - await self.update_testcase( + # update caserun + self.update_caserun( index=i, type='action', - end_time=datetime.now(), - passed=passed, + end_time=timezone.now(), + status=status, exception=exception, image=image - ) + ) + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + if step['action']['type'] == 'change': exception = None - passed = True - image = None - await self.update_testcase( + status = 'passed' + self.update_caserun( index=i, type='action', - start_time=datetime.now() + start_time=timezone.now() ) try: - print(f'changing element to value -> {step["action"]["value"]}') - # using puppeteer, find and click on the 'element' - if step["action"]["element"] != (None or ''): - selector = await self.format_element(step["action"]["element"]) - await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) - # scrolling to element using plain JavaScript - await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') - element = await self.page.J(selector) - await element.click(clickCount=3) - await self.page.keyboard.type(step["action"]["value"]) - time.sleep(int(self.configs['min_wait_time'])) + msg = f'changing element "{step["action"]["element"]["selector"]}" value to "{step["action"]["value"]}" | run_id: {str(self.caserun.id)}' + print(msg) + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + # using selenium, find and change the 'element'.value + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain javascript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # changing value of element + value = self.transpose_data(step["action"]["value"]) + element.send_keys(value) + time.sleep(int(self.configs.get('min_wait_time', 3))) + image = self.save_screenshot(run_type='run') except Exception as e: - image = await self.save_screenshot(page=self.page) - exception = e - passed = False + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) - await self.update_testcase( + # update caserun + self.update_caserun( index=i, type='action', - end_time=datetime.now(), - passed=passed, + end_time=timezone.now(), + status=status, exception=exception, image=image - ) + ) + + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + - if step['action']['type'] == 'keyDown': exception = None - passed = True - image = None - await self.update_testcase( + status = 'passed' + self.update_caserun( index=i, type='action', - start_time=datetime.now() + start_time=timezone.now() ) try: - print(f'keyDown action for key -> {step["action"]["key"]}') - # using puppeteer, press the selected key - await self.page.keyboard.press(step['action']['key']) - time.sleep(int(self.configs['min_wait_time'])) + msg = f'keyDown action using key "{step["action"]["key"]}" | run_id: {str(self.caserun.id)}' + print(msg) + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + # getting last known element + n = (i - 1) + elm = None + while True: + elm = self.steps[n]['action']['element']['selector'] + if elm != None and len(elm) != 0: + break + n -= 1 + selector = self.format_element(elm) + + # using selenium, find elemenmtn and send 'Key' event + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain javascript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # using selenium, press the selected key + element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"])) + time.sleep(int(self.configs.get('min_wait_time', 3))) + image = self.save_screenshot(run_type='run') except Exception as e: - image = await self.save_screenshot(page=self.page) - exception = e - passed = False + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' - await self.update_testcase( + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) + + # update caserun + self.update_caserun( index=i, type='action', - end_time=datetime.now(), - passed=passed, + end_time=timezone.now(), + status=status, exception=exception, image=image ) + + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + + if step['action']['type'] == 'switch': + exception = None + status = 'passed' + self.update_caserun( + index=i, type='action', + start_time=timezone.now() + ) + + try: + msg = f'switching to iframe element "{step["action"]["element"]["selector"]}" | run_id: {str(self.caserun.id)}' + print(msg) + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + + # using selenium, find and change the 'element'.value + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + # scrolling to element using plain javascript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # switching to iframe + self.driver.switch_to.frame(element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + image = self.save_screenshot(run_type='run') + + except Exception as e: + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) + + # update caserun + self.update_caserun( + index=i, type='action', + end_time=timezone.now(), + status=status, + exception=exception, + image=image + ) + + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break if step['assertion']['type'] == 'match': exception = None - passed = True - image = None - await self.update_testcase( + status = 'passed' + self.update_caserun( index=i, type='assertion', - start_time=datetime.now() + start_time=timezone.now() ) try: - print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}') - # using puppeteer, find elememt and assert if element.text == assertion.text - selector = await self.format_element(step["assertion"]["element"]) - await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) - # scrolling to element using plain JavaScript - await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()') - elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent') + # using selenium, find elememt and assert if element.text == assertion.value + msg = f'asserting that element "{step["assertion"]["element"]["selector"]}".innerText matches "{step["assertion"]["value"]}" | run_id: {str(self.caserun.id)}' + print(msg) + + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + selector = self.format_element(step["assertion"]["element"]["selector"]) + xpath = self.format_element(step["assertion"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # gettintg elem text + elementText = element.get_attribute('innerText') + elementText = element.text if len(elementText) == 0 else elementText elementText = elementText.strip() - print(f'elementText => {elementText}') - print(f'value => {step["assertion"]["value"]}') - assert elementText == step["assertion"]["value"] + print(f'elementText -> "{elementText}"') + print(f'value -> "{step["assertion"]["value"]}"') + + # assert text + if elementText != self.transpose_data(step["assertion"]["value"]): + raise AssertionError(f'innerText of element "{selector}" does match expected') + + # save screenshot + image = self.save_screenshot(run_type='run') except Exception as e: - image = await self.save_screenshot(page=self.page) - exception = e - passed = False + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) - await self.update_testcase( + # update caserun + self.update_caserun( index=i, type='assertion', - end_time=datetime.now(), - passed=passed, + end_time=timezone.now(), + status=status, exception=exception, image=image ) + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + if step['assertion']['type'] == 'exists': exception = None - passed = True - image = None - await self.update_testcase( + status = 'passed' + self.update_caserun( index=i, type='assertion', - start_time=datetime.now() + start_time=timezone.now() ) try: - print(f'asserting that element -> {step["assertion"]["element"]} exists') - # using puppeteer, find elememt and assert it exists - selector = await self.format_element(step["assertion"]["element"]) - await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000)) - await self.page.J(selector) + msg = f'asserting that {step["assertion"]["element"]["selector"]} exists | run_id: {str(self.caserun.id)}' + print(msg) + + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': msg + }) + + # find elememt and assert it exists + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element + self.driver.execute_script(self.scroll_to_center, element) + + # get step screenshot + image = self.save_screenshot(run_type='run') except Exception as e: - image = await self.save_screenshot(page=self.page) - exception = e - passed = False + image = self.save_screenshot(run_type='run') + exception = self.format_exception(e) + status = 'failed' - await self.update_testcase( + # updating flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': f'❌ {exception} | run_id: {str(self.caserun.id)}' + }) + + self.update_caserun( index=i, type='assertion', - end_time=datetime.now(), - passed=passed, + end_time=timezone.now(), + status=status, exception=exception, image=image ) - i += 1 - await self.update_testcase( - time_completed=datetime.now() + # exit early if configs.end_on_fail == True + if self.caserun.configs.get('end_on_fail', True) and status == 'failed': + break + + i += 1 + + self.update_caserun( + time_completed=timezone.now() ) - await self.driver.close() - print('-- testcase run complete --') \ No newline at end of file + quit_driver(driver=self.driver) + print('-- caserun run complete --') + + # update flowrun + if self.flowrun_id: + update_flowrun(**{ + 'flowrun_id': self.flowrun_id, + 'node_index': self.node_index, + 'message': ( + f'case run "{self.caserun.title}" for {self.caserun.site.site_url} completed with status: '+ + f'{"❌ FAILED" if self.caserun.status == 'failed' else "✅ PASSED"} | run_id: {str(self.caserun.id)}' + ), + 'objects': [{ + 'parent': str(self.caserun.site.id), + 'id': str(self.caserun.id), + 'status': self.caserun.status + }], + 'node_status': self.caserun.status + }) + + if self.caserun.status == 'failed' and self.caserun.configs.get('create_issue'): + print('generating new Issue...') + Issuer(caserun=self.caserun).build_issue() + + return None + + + + + def pre_run(self) -> None: + """ + Runs the self.case using selenium as the driver + and tries to collect element img & screenshot data. + + Returns: None + """ + + print(f'beginning pre_run for Case {self.case.title}') + + # setting implict wait_time for driver + self.driver.implicitly_wait(self.configs.get('max_wait_time')) + + i = 0 + for step in self.steps: + print(f'-- running step #{i+1} --') + + # adding catch if nav is not first + if i == 0 and step['action']['type'] != 'navigate': + print(f'navigating to {self.site_url} before first step') + # using selenium, navigate to site root path & wait for page to load + self.driver.get(f'{self.site_url}') + time.sleep(int(self.configs['min_wait_time'])) + + + if step['action']['type'] == 'navigate': + try: + print(f'navigating to {self.site_url}{step["action"]["path"]}') + # using selenium, navigate to requested path & wait for page to load + driver_wait( + driver=self.driver, + interval=int(self.configs.get('interval', 1)), + min_wait_time=int(self.configs.get('min_wait_time', 3)), + max_wait_time=int(self.configs.get('max_wait_time', 30)), + ) + self.driver.get(f'{self.site_url}{step["action"]["path"]}') + time.sleep(int(self.configs.get('min_wait_time', 3))) + + except Exception as e: + print(e) + + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['action']['image'] = img_url + + + if step['action']['type'] == 'scroll': + try: + print(f'scrolling -> {step["action"]["value"]}') + # scrolling using plain JavaScript + self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});') + time.sleep(int(self.configs.get('min_wait_time', 3))) + + except Exception as e: + print(e) + + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['action']['image'] = img_url + + + if step['action']['type'] == 'click': + try: + print(f'clicking element -> {step["action"]["element"]}') + # using selenium, find and click on the 'element' + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain JavaScript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # get elem img & update self.steps + if not self.steps[i]['action'].get('img'): + img = self.get_element_image(element) + self.steps[i]['action']['img'] = img + + # clicking element + element.click() + time.sleep(int(self.configs.get('min_wait_time', 3))) + + except Exception as e: + print(e) + + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['action']['image'] = img_url + + + if step['action']['type'] == 'mouseover': + try: + print(f'moving mouse to element -> {step["action"]["element"]}') + # using selenium, find and click on the 'element' + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain JavaScript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # get elem img & update self.steps + if not self.steps[i]['action'].get('img'): + img = self.get_element_image(element) + self.steps[i]['action']['img'] = img + + # moving mouse to element + self.actions.move_to_element(element).perform() + time.sleep(int(self.configs.get('min_wait_time', 3))) + + except Exception as e: + print(e) + + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['action']['image'] = img_url + + + if step['action']['type'] == 'change': + try: + print(f'changing element to value -> {step["action"]["value"]}') + # using selenium, find and change the 'element'.value + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain javascript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # get elem img & update self.steps + if not self.steps[i]['action'].get('img'): + img = self.get_element_image(element) + self.steps[i]['action']['img'] = img + + # changing value of element + value = step["action"]["value"] + element.send_keys(value) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + except Exception as e: + print(e) + + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['action']['image'] = img_url + + + if step['action']['type'] == 'keyDown': + try: + print(f'keyDown action for key -> {step["action"]["key"]}') + # getting last known element + n = (i - 1) + elm = None + while True: + elm = self.steps[n]['action']['element']['selector'] + if elm != None and len(elm) != 0: + break + n -= 1 + selector = self.format_element(elm) + + # using selenium, find element and send 'Key' event + selector = self.format_element(step["action"]["element"]["selector"]) + xpath = self.format_element(step["action"]["element"]["xpath"]) + element_data = self.get_element(selector, xpath) + element = element_data['element'] + + # checking if element was found + if element_data['failed']: + raise Exception(f'Unable to locate element with the given Selector and xPath') + + # scrolling to element using plain javascript + self.driver.execute_script(self.scroll_to_center, element) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + # get elem img & update self.steps + if not self.steps[i]['action'].get('img'): + img = self.get_element_image(element) + self.steps[i]['action']['img'] = img + + # using selenium, press the selected key + element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"])) + time.sleep(int(self.configs.get('min_wait_time', 3))) + + except Exception as e: + print(e) + + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['action']['image'] = img_url + + + if step['assertion']['type'] == 'match': + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['assertion']['image'] = img_url + + + if step['assertion']['type'] == 'exists': + # get screenshot and save + img_url = self.save_screenshot(run_type='pre_run') + self.steps[i]['assertion']['image'] = img_url + + + # increment step + i += 1 + + # update process + self.update_process( + current=(i+1), + total=self.case.steps['num_steps'], + complete=False + ) + + + # update case + steps_data = self.save_case_steps(self.steps, str(self.case.id)) + self.case.steps = steps_data + self.case.processed = True + self.case.save() + + quit_driver(driver=self.driver) + print('-- case pre_run complete --') + + # update process + self.update_process(current=1, total=1, complete=True) + + return None + + + + + + \ No newline at end of file diff --git a/app/api/utils/custom-config.js b/app/api/utils/configs/default-config.js similarity index 98% rename from app/api/utils/custom-config.js rename to app/api/utils/configs/default-config.js index cae0befc..a51b1005 100644 --- a/app/api/utils/custom-config.js +++ b/app/api/utils/configs/default-config.js @@ -1,7 +1,4 @@ // custom configurations for Lighthouse CLI - - - module.exports = { extends: 'lighthouse:default', plugins: ['lighthouse-plugin-crux'], diff --git a/app/api/utils/configs/extra-headers.json b/app/api/utils/configs/extra-headers.json new file mode 100644 index 00000000..e3f9597e --- /dev/null +++ b/app/api/utils/configs/extra-headers.json @@ -0,0 +1,5 @@ +{ + "Accept": "text/html", + "User-Agent": "Mozilla/5.0", + "Timing-Allow-Origin": "*" +} \ No newline at end of file diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py new file mode 100644 index 00000000..977b4c18 --- /dev/null +++ b/app/api/utils/crawler.py @@ -0,0 +1,183 @@ +from bs4 import BeautifulSoup +from .driver import * + + + + + + +class Crawler(): + """ + Crawl the passed "site" for pages, stoping + once 'max_urls' is reached. + + Args: + 'url' : str, + 'sitemap' : str, + 'max_urls' : int, + } + + Use `Crawler.get_links()` initiate a new crawl + + Returns: list + """ + + + + + def __init__(self, url: str=None, sitemap: str=None, max_urls: int=5): + self.url = url + self.sitemap = sitemap + self.max_urls = max_urls + self.driver = driver_init() + + + + + def get_links(self) -> list: + # crawl self.url and record any found links + # which are within the same self.url domain + + follow_urls = [] + crawled_urls = [] + saved_urls = [self.url,] + + + def url_is_valid(url: str=None) -> bool: + # checks if the passed url is + # a valid url to follow and + # not a file or external redirect + + bad_str_list = ['cdn-cgi'] + bad_end_list = [ + '.png', '.jpg', '.pdf', '.jpeg', + '.json', '.doc', '.svg', '.ppt', + '.pptx', '.ods', '.docx', '.mp3', + '.mp4', '.wma', '.ogg', '.mpa', + '.wpl', '.zip', '.pkg', '.tar.gz', + '.deb', '.z', '.rpm', '.7z', '.bin', + '.dmg', '.iso', '.toast', '.vcd', + '.csv', 'xml', '.db', '.dbf', '.dat', + '.log', '.mdb', '.sql', '.tar', '.sav', + '.webp', '.tiff', '.tif', '.psd', '.ps', + '.ico', '.gif', '.bmp' + ] + if not url.startswith(self.url) and not url.startswith('/'): + return False + for bad_str in bad_str_list: + if bad_str in url: + return False + for bad_end in bad_end_list: + if url.endswith(bad_end): + return False + return True + + + def crawl_url(start_url: str=None, max_depth: int=5): + + print(f'starting crawl on -> {start_url}') + + # adding url to list of crawled_urls + crawled_urls.append(start_url) + + # setting depth + depth = 0 + + # get requested start_url + self.driver.get(start_url) + + # wait for page to load + driver_wait( + driver=self.driver, + max_wait_time=20, + interval=2 + ) + + # parsing page_source + soup = BeautifulSoup(self.driver.page_source, 'html.parser') + + # iterating through all tags + + for link in soup.find_all('a'): + + # check if max_depth has been reached + if depth >= max_depth: + break + + url = link.get('href') + if url is not None: + # validate url + if url_is_valid(url): + if url.startswith('/'): + url = self.url + url + + # check status of page + self.driver.get(url) + + print(f'looped to this url -> {url}') + + # wait for page to load + resolved = driver_wait( + driver=self.driver, + max_wait_time=20, + interval=2 + ) + + # skipping url if not responding + if not resolved: + print('not resolved') + continue + + # clean and decide to record url + if str(self.driver.current_url) == str(url): + if url.endswith('/'): + url = url.rstrip('/') + if url not in follow_urls: + follow_urls.append(url) + depth += 1 + print(f'{depth} urls saved of {max_depth} allowed') + + + def record_urls(): + # adds all follow_urls to saved_urls + # if not already recorded + + max_reached = False + + # iterate through existing follow_urls + for url in follow_urls: + # pass if already crawled + if url not in crawled_urls: + if url not in saved_urls: + saved_urls.append(url) + print(f'saving -> {url}') + if len(saved_urls) >= self.max_urls: + print('max pages reached') + max_reached = True + break + return max_reached + + # layer 0 + crawl_url(self.url, max_depth=self.max_urls) + + # iterate through layers + while (len(follow_urls) > len(saved_urls)) and (len(saved_urls) < self.max_urls): + + # crawl each follow_url that + # has not been crawled + for url in follow_urls: + # add existing follow_urls first + max_reached = record_urls() + if max_reached: + break + + # crawl new url if not in crawled_urls + if url not in crawled_urls: + crawl_url(url, max_depth=self.max_urls) + + + # quit driver and return + quit_driver(self.driver) + return saved_urls + + diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py deleted file mode 100644 index 73601c4e..00000000 --- a/app/api/utils/crux.py +++ /dev/null @@ -1,36 +0,0 @@ -import requests, os, json - - - -class Crux(): - - def __init__(self, site_url): - self.site_url = site_url - self.key = os.environ.get('GOOGLE_CRUX_KEY') - - - def get_data(self): - - url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}' - headers = { - "Content-Type": "application/json", - } - data = { - "origin": str(self.site_url), - } - - res = requests.post( - url=url, - headers=headers, - data=json.dumps(data) - ) - - response = res.json() - - if res.status_code != 200: - response = { - "status": "failed", - "message": "This site_url does not have enough historical data in the CRUX API to respond with." - } - - return response diff --git a/app/api/utils/definitions.py b/app/api/utils/definitions.py new file mode 100644 index 00000000..1917d935 --- /dev/null +++ b/app/api/utils/definitions.py @@ -0,0 +1,415 @@ +# Data definitions used throughout +# the Cursion platform + + + + +definitions = [ + + # high-level test score + { + 'name': 'Test Score', + 'key': 'test_score', + 'value': 'obj.score' + }, + { + 'name': 'Health', + 'key': 'current_health', + 'value': 'obj.post_scan.score' + }, + { + 'name': 'Test Status', + 'key': 'test_status', + 'value': 'obj.status' + }, + { + 'name': 'VRT Score', + 'key': 'vrt_score', + 'value': 'obj.component_scores.get("vrt",0)' + }, + { + 'name': 'Logs Score', + 'key': 'logs_score', + 'value': 'obj.component_scores.get("logs",0)' + }, + { + 'name': 'HTML Score', + 'key': 'html_score', + 'value': 'obj.component_scores.get("html",0)' + }, + { + 'name': 'Yellowlab Score', + 'key': 'yellowlab_score', + 'value': 'obj.component_scores.get("yellowlab",0)' + }, + { + 'name': 'Lighthouse Score', + 'key': 'lighthouse_score', + 'value': 'obj.component_scores.get("lighthouse",0)' + }, + + # high-level scan score + { + 'name': 'Health', + 'key': 'health', + 'value': 'obj.score' + }, + { + 'name': 'Error Logs', + 'key': 'logs', + 'value': 'len(obj.logs)' + }, + + # LH test data + { + 'name': 'SEO Delta', + 'key': 'seo_delta', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("seo_delta",0)' + }, + { + 'name': 'PWA Delta', + 'key': 'pwa_delta', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("pwa_delta",0)' + }, + { + 'name': 'CRUX Delta', + 'key': 'crux_delta', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("crux_delta",0)' + }, + { + 'name': 'Best Practices Delta', + 'key': 'best_practices_delta', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("best_practices_delta",0)' + }, + { + 'name': 'Performance Delta', + 'key': 'performance_delta', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("performance_delta",0)' + }, + { + 'name': 'Accessibility Delta', + 'key': 'accessibility_delta', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("accessibility_delta",0)' + }, + { + 'name': 'Lighthouse Average', + 'key': 'current_lighthouse_average', + 'value': '((obj.lighthouse_delta or {}).get("scores") or {}).get("current_average",0)' + }, + + # LH scan data + { + 'name': 'Lighthouse Average', + 'key': 'lighthouse_average', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("average",0)' + }, + { + 'name': 'SEO', + 'key': 'seo', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("seo",0)' + }, + { + 'name': 'PWA', + 'key': 'pwa', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("pwa",0)' + }, + { + 'name': 'CRUX', + 'key': 'crux', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("crux",0)' + }, + { + 'name': 'Best Practice', + 'key': 'best_practices', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("best_practices",0)' + }, + { + 'name': 'Performance', + 'key': 'performance', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("performance",0)' + }, + { + 'name': 'Accessibility', + 'key': 'accessibility', + 'value': '((obj.lighthouse or {}).get("scores") or {}).get("accessibility",0)' + }, + + # YL test data + { + 'name': 'Yellowlab Average', + 'key': 'current_yellowlab_average', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("current_average",0)' + }, + { + 'name': 'Page Weight Delta', + 'key': 'pageWeight_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("pageWeight_delta",0)' + }, + { + 'name': 'Images Delta', + 'key': 'images_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("images_delta",0)' + }, + { + 'name': ' DOM Complexity Delta', + 'key': 'domComplexity_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("domComplexity_delta",0)' + }, + { + 'name': 'JS Complexity Delta', + 'key': 'javascriptComplexity_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("javascriptComplexity_delta",0)' + }, + { + 'name': 'Bad JS Delta', + 'key': 'badJavascript_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("badJavascript_delta",0)' + }, + { + 'name': 'jQuery Delta', + 'key': 'jQuery_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("jQuery_delta",0)' + }, + { + 'name': 'CSS Complexity Delta', + 'key': 'cssComplexity_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("cssComplexity_delta",0)' + }, + { + 'name': 'Bad CSS Delta', + 'key': 'badCSS_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("badCSS_delta",0)' + }, + { + 'name': 'Fonts Delta', + 'key': 'fonts_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("fonts_delta",0)' + }, + { + 'name': 'Server Config Delta', + 'key': 'serverConfig_delta', + 'value': '((obj.yellowlab_delta or {}).get("scores") or {}).get("serverConfig_delta",0)' + }, + + # YL scan data + { + 'name': 'Yellowlab Average', + 'key': 'yellowlab_average', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("globalScore",0)' + }, + { + 'name': 'Page Weight', + 'key': 'pageWeight', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("pageWeight",0)' + }, + { + 'name': 'Images', + 'key': 'images', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("images",0)' + }, + { + 'name': 'DOM Complexity', + 'key': 'domComplexity', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("domComplexity",0)' + }, + { + 'name': 'JS Complexity', + 'key': 'javascriptComplexity', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("javascriptComplexity",0)' + }, + { + 'name': 'Bad JS', + 'key': 'badJavascript', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("badJavascript",0)' + }, + { + 'name': 'jQuery', + 'key': 'jQuery', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("jQuery",0)' + }, + { + 'name': 'CSS Complexity', + 'key': 'cssComplexity', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("cssComplexity",0)' + }, + { + 'name': 'Bad CSS', + 'key': 'badCSS', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("badCSS",0)' + }, + { + 'name': 'Fonts', + 'key': 'fonts', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("fonts",0)' + }, + { + 'name': 'Server Configs', + 'key': 'serverConfig', + 'value': '((obj.yellowlab or {}).get("scores") or {}).get("serverConfig",0)' + }, + + # caserun + { + 'name': 'Case Run Status', + 'key': 'caserun_status', + 'value': 'obj.status' + }, + { + 'name': 'Case Run ID', + 'key': 'caserun_id', + 'value': 'str(obj.id)' + }, + { + 'name': 'Case Title', + 'key': 'case_title', + 'value': 'obj.title' + }, + { + 'name': 'Case ID', + 'key': 'case_id', + 'value': 'str(obj.case.id)' + }, + + # flowrun + { + 'name': 'Flow Run Status', + 'key': 'flowrun_status', + 'value': 'obj.status' + }, + { + 'name': 'Flow Run ID', + 'key': 'flowrun_id', + 'value': 'str(obj.id)' + }, + { + 'name': 'Flow Title', + 'key': 'flow_title', + 'value': 'obj.title' + }, + { + 'name': 'Flow ID', + 'key': 'flow_id', + 'value': 'str(obj.flow.id)' + }, + + # report + { + 'name': 'Report URL', + 'key': 'report_url', + 'value': 'obj.path' + }, + { + 'name': 'Report ID', + 'key': 'report_id', + 'value': 'str(obj.id)' + }, + + # issue + { + 'name': 'Issue Title', + 'key': 'issue_title', + 'value': 'obj.title' + }, + { + 'name': 'Issue Details', + 'key': 'issue_details', + 'value': 'obj.details' + }, + { + 'name': 'Issue ID', + 'key': 'issue_id', + 'value': 'str(obj.id)' + }, + { + 'name': 'Issue Affected ID', + 'key': 'issue_affected_id', + 'value': 'str(obj.affected.get("id"))' + }, + { + 'name': 'Issue Affected', + 'key': 'issue_affected', + 'value': 'str(obj.affected.get("str"))' + }, + { + 'name': 'Issue Affected Type', + 'key': 'issue_affected_type', + 'value': 'str(obj.affected.get("type"))' + }, + { + 'name': 'Issue Trigger Type', + 'key': 'issue_trigger_type', + 'value': 'str(obj.trigger.get("type"))' + }, + { + 'name': 'Issue Trigger ID', + 'key': 'issue_trigger_id', + 'value': 'str(obj.trigger.get("id"))' + }, + + # test + { + 'name': 'Test ID', + 'key': 'test_id', + 'value': 'str(obj.id)' + }, + + # scan + { + 'name': 'Scan ID', + 'key': 'scan_id', + 'value': 'str(obj.id)' + }, + + # page + { + 'name': 'Page ID', + 'key': 'page_id', + 'value': 'str(obj.page.id)' + }, + { + 'name': 'Page URL', + 'key': 'page_url', + 'value': 'obj.page.page_url' + }, + + # site + { + 'name': 'Site ID', + 'key': 'site_id', + 'value': 'str(obj.site.id)' + }, + { + 'name': 'Site URL', + 'key': 'site_url', + 'value': 'obj.site.site_url' + }, +] + + + + + +# get definition +def get_definition(key: str=None, name: str=None) -> str: + """ + Finds the specific data definition based on the + key or name provided. + + Args: + "key" : str, + "name" : str, + + Returns: "definition" dict, or None + """ + + # setting default + selected = None + + # iterate and search through definitions + for obj in definitions: + if obj['key'] == key or obj['name'] == name: + selected = obj + break + + # return definition + return selected \ No newline at end of file diff --git a/app/api/utils/devices.py b/app/api/utils/devices.py new file mode 100644 index 00000000..58f812cc --- /dev/null +++ b/app/api/utils/devices.py @@ -0,0 +1,255 @@ +# This is a ChatGPT generated list of devices +# https://chatgpt.com/c/6706e3f9-d6f0-8004-8564-12947cc76e2b + + + + +devices = [ + { + "id": "1", + "name": "Google Pixel 5", + "window_size": "393,851", + "user_agent": "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36", + "browser": "chrome", + "type": "mobile" + }, + { + "id": "2", + "name": "Samsung Galaxy S21", + "window_size": "412,915", + "user_agent": "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36", + "browser": "chrome", + "type": "mobile" + }, + { + "id": "3", + "name": "iPhone 12 Pro", + "window_size": "390,844", + "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "browser": "chrome", + "type": "mobile" + }, + { + "id": "4", + "name": "iPad Pro", + "window_size": "1024,1366", + "user_agent": "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "browser": "chrome", + "type": "tablet" + }, + { + "id": "5", + "name": "Samsung Galaxy Tab S7", + "window_size": "800,1280", + "user_agent": "Mozilla/5.0 (Linux; Android 10; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "browser": "chrome", + "type": "tablet" + }, + { + "id": "6", + "name": "MacBook Pro 16", + "window_size": "1536,960", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "browser": "chrome", + "type": "desktop" + }, + { + "id": "7", + "name": "Windows 10 PC", + "window_size": "1920,1080", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "browser": "chrome", + "type": "desktop" + }, + { + "id": "8", + "name": "iMac 24-inch", + "window_size": "2560,1440", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "browser": "chrome", + "type": "desktop" + }, + { + "id": "9", + "name": "Chromebook Pixel", + "window_size": "1280,850", + "user_agent": "Mozilla/5.0 (X11; CrOS x86_64 14092.54.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "browser": "chrome", + "type": "desktop" + }, + { + "id": "10", + "name": "Google Pixel 5", + "window_size": "393,851", + "user_agent": "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Mobile Safari/537.36", + "browser": "firefox", + "type": "mobile" + }, + { + "id": "11", + "name": "Samsung Galaxy S21", + "window_size": "412,915", + "user_agent": "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Mobile Safari/537.36", + "browser": "firefox", + "type": "mobile" + }, + { + "id": "12", + "name": "iPhone 12 Pro", + "window_size": "390,844", + "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "browser": "firefox", + "type": "mobile" + }, + { + "id": "13", + "name": "iPad Pro", + "window_size": "1024,1366", + "user_agent": "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "browser": "firefox", + "type": "tablet" + }, + { + "id": "14", + "name": "Samsung Galaxy Tab S7", + "window_size": "800,1280", + "user_agent": "Mozilla/5.0 (Linux; Android 10; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36", + "browser": "firefox", + "type": "tablet" + }, + { + "id": "15", + "name": "MacBook Pro 16", + "window_size": "1536,960", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36", + "browser": "firefox", + "type": "desktop" + }, + { + "id": "16", + "name": "Windows 10 PC", + "window_size": "1920,1080", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36", + "browser": "firefox", + "type": "desktop" + }, + { + "id": "17", + "name": "iMac 24-inch", + "window_size": "2560,1440", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36", + "browser": "firefox", + "type": "desktop" + }, + { + "id": "18", + "name": "Chromebook Pixel", + "window_size": "1280,850", + "user_agent": "Mozilla/5.0 (X11; CrOS x86_64 14092.54.0) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/113.0.0 Safari/537.36", + "browser": "firefox", + "type": "desktop" + }, + { + "id": "19", + "name": "Google Pixel 5", + "window_size": "393,851", + "user_agent": "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Mobile Safari/537.36", + "browser": "edge", + "type": "mobile" + }, + { + "id": "20", + "name": "Samsung Galaxy S21", + "window_size": "412,915", + "user_agent": "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Mobile Safari/537.36", + "browser": "edge", + "type": "mobile" + }, + { + "id": "21", + "name": "iPhone 12 Pro", + "window_size": "390,844", + "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "browser": "edge", + "type": "mobile" + }, + { + "id": "22", + "name": "iPad Pro", + "window_size": "1024,1366", + "user_agent": "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "browser": "edge", + "type": "tablet" + }, + { + "id": "23", + "name": "Samsung Galaxy Tab S7", + "window_size": "800,1280", + "user_agent": "Mozilla/5.0 (Linux; Android 10; SM-T870) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36", + "browser": "edge", + "type": "tablet" + }, + { + "id": "24", + "name": "MacBook Pro 16", + "window_size": "1536,960", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36", + "browser": "edge", + "type": "desktop" + }, + { + "id": "25", + "name": "Windows 10 PC", + "window_size": "1920,1080", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36", + "browser": "edge", + "type": "desktop" + }, + { + "id": "26", + "name": "iMac 24-inch", + "window_size": "2560,1440", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36", + "browser": "edge", + "type": "desktop" + }, + { + "id": "27", + "name": "Chromebook Pixel", + "window_size": "1280,850", + "user_agent": "Mozilla/5.0 (X11; CrOS x86_64 14092.54.0) AppleWebKit/537.36 (KHTML, like Gecko) Edg/113.0.0.0 Safari/537.36", + "browser": "edge", + "type": "desktop" + } +] + + + + +# get device +def get_device(browser: str=None, name: str=None) -> str: + """ + Finds the specific device based on the + browser and name provided. + + Args: + "browser": str, + "name": str, + + Returns: "device" dict + """ + + # setting default to 'Windows 10 PC' + selected = devices[6] + + # iterate and search through devices + for device in devices: + if device['browser'] == browser and device['name'] == name: + selected = device + break + + # return device + return selected + + + diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py new file mode 100644 index 00000000..786cd4d7 --- /dev/null +++ b/app/api/utils/driver.py @@ -0,0 +1,359 @@ +from selenium import webdriver +from selenium.webdriver.common.actions.action_builder import ActionBuilder +from selenium.webdriver.firefox.options import Options +from selenium.webdriver.firefox.firefox_profile import FirefoxProfile +from .devices import get_device +from datetime import datetime +import time, os, sys, tempfile + + + + + + +def driver_init( + browser : str='chrome', + window_size : str='1920,1080', + device : str='Windows 10 PC', + pixel_ratio : int=1.0, + scale_factor : int=0.5 + ) -> object: + """ + Starts a new selenium driver instance + + Args: + 'browser' : str, + 'window_size' : str, + 'device' : str, + 'script_timeout': int, + 'load_timeout' : int, + 'wait_time' : int, + 'pixel_ratio' : int, + 'scale_factor' : int + + Returns: driver object + """ + + # get userAgent + user_agent = get_device(browser, device)['user_agent'] + + # deciding on browser + # UserAgents are from utils/devices + if browser == 'chrome': + options = webdriver.ChromeOptions() + options.binary_location = os.environ.get('CHROME_BROWSER') + if browser == 'firefox': + options = webdriver.FirefoxOptions() + options.binary_location = os.environ.get('FIREFOX_BROWSER') + if browser == 'edge': + options = webdriver.EdgeOptions() + options.binary_location = os.environ.get('EDGE_BROWSER') + + # setting up browser configs + sizes = window_size.split(',') + width = int(sizes[0]) + height = int(sizes[1]) + emulation = { + "deviceMetrics": { + "width": width, + "height": height, + "pixelRatio": pixel_ratio + }, + "userAgent": user_agent + } + + # setting broswer options for chrome + if browser == 'chrome': + options.add_argument("--no-sandbox") + options.add_argument("--disable-gpu") + options.add_argument("--disable-gpu-compositing") + options.add_argument("--use-gl=swiftshader") + options.add_argument("--disable-blink-features=AutomationControlled") + options.add_argument("--headless") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--enable-unsafe-swiftshader") + options.add_argument("--ignore-certificate-errors") + options.add_argument("--hide-scrollbars") + options.add_argument(f"--force-device-scale-factor={str(scale_factor)}") + options.add_argument(f"--user-agent={user_agent}") + options.set_capability("goog:loggingPrefs", {"performance": "ALL"}) + options.page_load_strategy = 'none' + + # setting to mobile or tablet if reqeusted + if device == 'mobile' or device == 'tablet': + options.add_experimental_option("mobileEmulation", emulation) + + # init driver + driver = webdriver.Chrome(options=options) + + # setting broswer options for firefox + if browser == 'firefox': + # setting profile + temp_profile_dir = tempfile.mkdtemp() + ff_profile = FirefoxProfile(temp_profile_dir) + # adding arguments + options.add_argument("-headless") + options.page_load_strategy = 'none' + options.set_preference("accept_insecure_certs", True) + options.set_preference('layout.css.devPixelsPerPx', str(scale_factor)) + options.profile = ff_profile + + # setting to mobile if reqeusted + if device == 'mobile': + options.set_preference( + "general.useragent.override", f"userAgent={user_agent}" + ) + + # init driver + driver = webdriver.Firefox(options=options) + + # setting broswer options for edge + if browser == 'edge': + options.add_argument("--no-sandbox") + options.add_argument("--disable-gpu") + options.add_argument("--disable-gpu-compositing") + options.add_argument("--use-gl=swiftshader") + options.add_argument("--disable-blink-features=AutomationControlled") + options.add_argument("--headless") + options.add_argument("--enable-unsafe-swiftshader") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--ignore-certificate-errors") + options.add_argument("--hide-scrollbars") + options.add_argument(f"--force-device-scale-factor={str(scale_factor)}") + options.add_argument(f"--user-agent={user_agent}") + options.set_capability("goog:loggingPrefs", {"performance": "ALL"}) + options.page_load_strategy = 'none' + + # setting to mobile or tablet if reqeusted + if device == 'mobile' or device == 'tablet': + options.add_experimental_option("mobileEmulation", emulation) + + # init driver + driver = webdriver.Edge(options=options) + + # resizing window + driver.maximize_window() + driver.set_window_size(width, height) + print(f'Using {browser} browser') + + return driver + + + + +def driver_test() -> None: + """ + Spins up a selenium driver instance and + tests to ensure it can access the browser and internet + + Returns: None + """ + + print("Testing Selenium...") + message = 'Selenium was unable to start\n\n' + status = 'Failed' + + # testing selenium + try: + driver = driver_init() + driver.set_page_load_timeout(20) + driver.get('https://google.com') + title = driver.title + assert title == 'Google' + if title == 'Google': + status = 'Success' + message = 'Selenium installed and working \N{check mark} \n\n' + # log exception + except Exception as e: + print(e) + + # logging test results + sys.stdout.write( + '--- ' + status + ' ---\n'+ message + ) + + try: + quit_driver(driver) + sys.exit(0) + except: + pass + + return None + + + + +def driver_wait( + driver: object, + interval: int=1, + max_wait_time: int=30, + min_wait_time: int=3 + ) -> bool: + """ + Expects the driver instance and waits + for either the page to fully load or the max_wait_time + to expire before returning. + + Args: + 'driver' : object, + 'interval' : int, + 'max_wait_time' : int, + 'min_wait_time' : int + + Returns: bool (True if page is loaded) + """ + + def interact_with_page(driver): + # simulate mouse movement + action = ActionBuilder(driver) + action.pointer_action.move_to_location(0, 0) + action.perform() + # wait for 1s + time.sleep(0.5) + action.pointer_action.move_to_location(0, 50) + action.perform() + # wait for 1s + time.sleep(0.5) + action.pointer_action.move_to_location(0, 0) + action.perform() + return + + resolved = False + page_state = 'loading' + wait_time = 0 + + # min_wait_time before checking page status + time.sleep(int(min_wait_time)) + + while int(wait_time) < int(max_wait_time) and page_state != 'complete': + + # get current timestamp + pre_check_time = datetime.now() + + # wait 1 sec or sec + time.sleep(int(interval)) + + try: + page_state = driver.execute_script('return document.readyState') + except Exception as e: + print(e) + + # get time after waiting for script + post_check_time = datetime.now() + + # get seconds between checks + time_to_add = (post_check_time - pre_check_time).total_seconds() + + print(f'document state is {page_state}') + if page_state == 'complete': + resolved = True + + wait_time += time_to_add + + # interacting with page if available + if resolved: + interact_with_page(driver) + + return resolved + + + + +def get_data( + driver: object, + browser: str='chrome', + interval: int=1, + max_wait_time: int=30, + min_wait_time: int=3 + ) -> dict: + """ + Once the page has loaded, grabs the + page-source (html) and console-logs (logs). + + Args: + 'driver' : object, + 'browser' : str, + 'interval' : int, + 'max_wait_time' : int, + 'min_wait_time' : int + + Returns: + 'html' : str, + 'logs' : dict + """ + + # setting defaults + html = None + logs = [] + + # waiting for page to load + driver_wait( + driver=driver, + interval=interval, + max_wait_time=max_wait_time, + min_wait_time=min_wait_time + ) + + def is_ignorable_warning(log_entry: object) -> bool: + ignore_list = [ + "WebGL", "GL Driver Message", "GPU", "No available adapters", + ] + try: + message = (log_entry or {}).get("message", "") + except Exception: + return False + for i in ignore_list: + if i in message: + return True + return False + + # get page_source from browser + try: + html = driver.page_source + except Exception as e: + print(e) + + # get console logs if notn firefox + if browser != 'firefox' : + try: + logs = driver.get_log('browser') + logs = [entry for entry in logs if not is_ignorable_warning(entry)] + except Exception as e: + print(e) + + # formatting respones + data = { + "html": html, + "logs": logs + } + + return data + + + + +def quit_driver(driver: object) -> None: + """ + Quits and reaps all child processes in docker + + Returns: None + """ + print('Quitting session: %s' % driver.session_id) + driver.quit() + try: + pid = True + while pid: + pid = os.waitpid(-1, os.WNOHANG) + print("Reaped child: %s" % str(pid)) + # avoid infinite loop cause pid value -> (0, 0) + try: + if pid[0] == 0: + pid = False + except: + pass + except ChildProcessError: + pass + + + + diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py deleted file mode 100644 index c9ccce82..00000000 --- a/app/api/utils/driver_p.py +++ /dev/null @@ -1,175 +0,0 @@ -from pyppeteer import launch -import time, os, numpy, json, sys, datetime, asyncio - - - -async def driver_init( - window_size='1920,1080', - wait_time=30, - ): - - sizes = window_size.split(',') - - options = { - 'executablePath': os.environ.get('CHROMIUM'), - 'args': [ - '--no-sandbox', - '--disable-dev-shm-usage', - f'--window-size={window_size}', - ], - 'defaultViewport': { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - }, - 'timeout': wait_time * 1000 - } - - driver = await launch( - options=options, - headless=True, - handleSIGINT=False, - handleSIGTERM=False, - handleSIGHUP=False - ) - - return driver - - - - - -async def interact_with_page(page): - # simulate mouse movement - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - - return page - - - - - - -async def driver_test(*args, **options): - - print("Testing puppeteer instalation and integration...") - - try: - driver = await driver_init() - page = await driver.newPage() - await page.goto('https://google.com', {'waitUntil': 'networkidle0'}) - await interact_with_page(page) - title = await page.title() - assert title == 'Google' - if title == 'Google': - status = 'Success' - else: - status = 'Failed' - await driver.close() - except Exception as e: - print(e) - status = 'Failed' - - sys.stdout.write('--- ' + status + ' ---\n' - + 'Puppeteer installed and working \N{check mark} \n' - ) - - - - - - -async def get_data(url, configs, *args, **options): - sizes = configs['window_size'].split(',') - driver = await driver_init(window_size=configs['window_size']) - page = await driver.newPage() - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs['max_wait_time']*1000 - } - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36" - ) - - await page.setViewport(viewport) - - if configs['device'] == 'mobile': - await page.setUserAgent(userAgent) - - - logs = [] - def record_logs(log): - if log.type == 'error': - if '.js' in log.text: - source = 'javascript' - elif 'http' in log.text: - source = 'network' - else: - source = 'other' - log_obj = { - "level": "SEVERE", - "source": source, - "message": str(log.text), - "timestamp": int(datetime.datetime.now().timestamp() * 1000) - } - logs.append(log_obj) - elif log.type == 'warning': - if '.js' in log.text: - source = 'javascript' - elif 'http' in log.text: - source = 'network' - else: - source = 'other' - log_obj = { - "level": "WARNING", - "source": source, - "message": str(log.text), - "timestamp": int(datetime.datetime.now().timestamp() * 1000) - } - logs.append(log_obj) - - def record_network(request): - log_obj = { - "level": "SEVERE", - "source": "network", - "message": f'{request.failure()["errorText"]} {request.url}', - "timestamp": int(datetime.datetime.now().timestamp() * 1000) - } - logs.append(log_obj) - - def record_error(error): - err = str(error).split(' at ')[0] - log_obj = { - "level": "SEVERE", - "source": "javascript", - "message": f'{err}', - "timestamp": int(datetime.datetime.now().timestamp() * 1000) - } - logs.append(log_obj) - - - page.on('console', lambda log : record_logs(log)) - page.on('requestfailed', lambda request : record_network(request)) - page.on('pageerror', lambda error : record_error(error)) - - await page.goto(url, page_options) - - # await page.waitForNavigation(navWaitOpt) - await interact_with_page(page) - html = await page.content() - - await driver.close() - - data = { - 'html': html, - 'logs': logs, - } - - return data \ No newline at end of file diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py deleted file mode 100644 index 582c3b0e..00000000 --- a/app/api/utils/driver_s.py +++ /dev/null @@ -1,167 +0,0 @@ -from selenium import webdriver -from selenium.webdriver.common.desired_capabilities import DesiredCapabilities -from selenium.webdriver import ActionChains -import time, os, numpy, json, sys - - - -def driver_init( - window_size='1920,1080', - device='desktop', - script_timeout=30, - load_timeout=30, - wait_time=15, - ): - - sizes = window_size.split(',') - - prefs = { - 'download.prompt_for_download': False, - 'download.extensions_to_open': '.zip', - 'safebrowsing.enabled': True - } - - mobile_emulation = { - "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 }, - "userAgent": ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36" - ) - } - - chromedriver_path = os.environ.get("CHROMEDRIVER") - options = webdriver.ChromeOptions() - options.binary_location = os.environ.get('CHROMIUM') - options.add_argument("--no-sandbox") - options.add_argument("disable-blink-features=AutomationControlled") - options.add_experimental_option('prefs',prefs) - options.add_argument("start-maximized") - options.add_argument("--headless") - options.add_argument("--disable-dev-shm-usage") - options.add_argument("--window-size=%s" % window_size) - - if device == 'mobile': - options.add_experimental_option("mobileEmulation", mobile_emulation) - - caps = DesiredCapabilities.CHROME - caps['goog:loggingPrefs'] = {'performance': 'ALL'} - - driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps) - driver.set_page_load_timeout(load_timeout) - driver.set_script_timeout(script_timeout) - driver.implicitly_wait(wait_time) - - - return driver - - -def driver_test(): - - print("Testing selenium instalation and integration...") - try: - driver = driver_init() - driver.get('https://google.com') - title = driver.title - assert title == 'Google' - if title == 'Google': - status = 'Success' - else: - status = 'Failed' - except Exception as e: - print(e) - status = 'Failed' - - sys.stdout.write('--- ' + status + ' ---\n' - + 'Selenium installed and working \N{check mark} \n' - ) - - quit_driver(driver) - sys.exit(0) - - - -def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5): - """ - Pauses the driver until all network requests have been resolved - - --> Adding mouse interaction to load WP plugin rendered content - - returns once driver determines that all request have resolved or - total wait time exceeds max_wait_time - - """ - - def get_request_list(driver): - # get current snapshot of driver requests - requests = driver.get_log('performance') - r_list = [] - for r in requests: - network_log = json.loads(r["message"])["message"] - - # Checks if the current 'method' key has any Network related value. - if("Network.response" in network_log["method"] - or "Network.request" in network_log["method"] - or "Network.webSocket" in network_log["method"]): - - r_list.append(network_log) - - return r_list - - - def interact_with_page(driver): - # simulate mouse movement and click on tag - html_tag = driver.find_elements_by_tag_name('html')[0] - action = ActionChains(driver) - action.move_to_element(html_tag).perform() - return - - - resolved = False - wait_time = 0 - - # actions before comparing network logs - interact_with_page(driver) - time.sleep(min_wait_time) - - while not resolved and wait_time < max_wait_time: - # get first set of logs - list_one = get_request_list(driver=driver) - - # wait 5 sec or sec for request to resolve - time.sleep(interval) - - # get second set of logs - list_two = get_request_list(driver=driver) - - # check if logs are equal - resolved = numpy.array_equal(list_one, list_two) - - wait_time += interval - - return - - - - -def quit_driver(driver): - ''' - Quits and reaps all child processes in docker - ''' - print('Quitting session: %s' % driver.session_id) - driver.quit() - try: - pid = True - while pid: - pid = os.waitpid(-1, os.WNOHANG) - print("Reaped child: %s" % str(pid)) - - # avoid infinite loop cause pid value -> (0, 0) - try: - if pid[0] == 0: - pid = False - except: - pass - - - except ChildProcessError: - pass \ No newline at end of file diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py new file mode 100644 index 00000000..e1f114b4 --- /dev/null +++ b/app/api/utils/exporter.py @@ -0,0 +1,126 @@ +from .driver import driver_init, driver_wait, quit_driver +from PIL import Image as I +from .alerts import sendgrid_email +from cursion import settings +import time, boto3, os + + + + + +def create_and_send_report_export(report_id: id, email: str, first_name: str) -> dict: + """ + Takes a screenshot of the `landing.report`, + save as a PDF, upload to s3 bucket, and then + send an email to the prospect that requested it. + + Expects the following: + 'report_id' : id of report/page being reported on + 'email' : str prospect's email address + 'first_name' : str prospect's first name + + Returns: + 'success' : bool if process started successfully + 'error' : str any error msg from cursion server + """ + + # setting up s3 client + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # init driver + driver = driver_init(scale_factor=1) + + # nav to report page + driver.get(f'{settings.LANDING_URL_ROOT}/report/{report_id}') + time.sleep(5) + + # wait for page to load + driver_wait(driver=driver) + + # setting screensize + full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;") + driver.set_window_size(1260, int(full_page_height)) # 1512 x full_page + + # taking screenshot + driver.save_screenshot(f'{report_id}.png') + + # quitting driver + quit_driver(driver) + + # setting up paths + image = os.path.join(settings.BASE_DIR, f'{report_id}.png') + pdf = os.path.join(settings.BASE_DIR, f'{report_id}.pdf') + + # resizing image to remove excess | expected height => 2353 + img = I.open(image) + width, height = img.size + + # Validate crop bounds + left = 0 + top = 85 + right = width + bottom = max(0, height - 330) + + cropped_img = img.crop((left, top, right, bottom)) + cropped_img.save(image, quality=95) + + # Convert to PDF + img = I.open(image) + pdf_img = img.convert('RGB') + pdf_img.save(pdf) + + # uploading to s3 + remote_path = f'static/landing/reports/{report_id}.pdf' # -> .png + report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}' + + # upload to s3 + with open(pdf, 'rb') as data: # -> image + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'} + ) + + # removing local copies + os.remove(image) + os.remove(pdf) + + # setting up email to prospect + pre_content = 'The Cursion performance report you requested has finished processing. \ + Now, just click the link below to view and download the PDF.' + content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://cursion.dev/booking' + subject = f'{first_name}, your Cursion Report is Ready' + title = f'{first_name}, your Cursion Report is Ready' + pre_header = f'{first_name}, your Cursion Report is Ready' + button_text = 'View Your Report' + email = email + object_url = report_url + signature = f'- Landon R | CEO @Cursion' + greeting = f'Hi {first_name},' + + message_obj = { + 'pre_content': pre_content, + 'content': content, + 'subject': subject, + 'title': title, + 'pre_header': pre_header, + 'button_text': button_text, + 'email': email, + 'object_url': object_url, + 'signature': signature, + 'greeting': greeting + } + + # sending email to prospect + data = sendgrid_email(message_obj=message_obj) + + # returning data + return data + + + + + diff --git a/app/api/utils/flowr.py b/app/api/utils/flowr.py new file mode 100644 index 00000000..cc1bf610 --- /dev/null +++ b/app/api/utils/flowr.py @@ -0,0 +1,968 @@ +from ..models import * +from .alerter import Alerter +from ..tasks import ( + create_caserun_bg, create_report_bg, + create_scan_bg, create_test_bg, + create_issue_bg, + send_phone_bg, send_email_bg, + send_slack_bg, send_webhook_bg +) +from django.utils import timezone +from django.core.cache import cache +from datetime import datetime +import time, uuid, random + + + + + + +class Flowr(): + """ + Executes a `FlowRun` based on the state of + the `FlowRun` instance. + + Use `Flowr.run_next()` to run next step in `FlowRun` + + Args: + - 'flowrun_id' str + + Returns: + - Flow instance + """ + + + + + def __init__(self, flowrun_id: str=None) -> object: + + # retrieve flowrun + self.flowrun_id = flowrun_id + self.flowrun = FlowRun.objects.get(id=flowrun_id) + + # constants for tasks that require 'object_id' + self.alert_types = [ + 'webhook', 'email', 'phone', 'slack', 'report', 'issue' + ] + + + + + def build_timestamp(self) -> object: + # build timestamp + return timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f') + + + + + def get_timestamp(self, timestamp: str=None) -> object: + """ + Formats the 'timestamp' if not None + + Args: + timestamp: str + } + + Returns: datetime object + """ + + # format for timestamp + f = '%Y-%m-%d %H:%M:%S.%f' + + if timestamp: + # clean timestamp str + clean_str = timestamp.replace('T', ' ').replace('Z', '') + # format date str as datetime obj + return datetime.strptime(clean_str, f) + + # return None if no timestamp + return None + + + + + def get_current_step(self) -> dict: + """ + Finds the most recently completed step + + Expects: None + + Returns: + 'index' : int, + 'node' : dict + } + """ + + # copy all "completed" self.flowrun.nodes + nodes = [ + node for node in self.flowrun.nodes \ + if (node['data']['time_completed'] and not node['data']['finalized']) + ] + + # sort nodes/steps by time_completed + sorted_nodes = sorted( + nodes, + key=lambda x: self.get_timestamp(x['data']['time_completed']), + reverse=True + ) + + # get current_node + current_node = sorted_nodes[0] if len(sorted_nodes) > 0 else None + + # get index of current_node + index = 0 + if current_node: + for node in self.flowrun.nodes: + if current_node['id'] == node['id']: + break + index+=1 + + # return data + data = { + 'index': index, + 'node': current_node + } + return data + + + + + def get_last_node_id(self) -> str: + """ + Sorts the nodes by time_completed + and largets ID. + + Expects: None + + Returns: ID + """ + # copy all "completed" self.flowrun.nodes + nodes = [ + node for node in self.flowrun.nodes if (node['data']['time_completed']) + ] + + # sort nodes/steps by decending int(id) + sorted_nodes = sorted( + nodes, + key=lambda x: int(x['data']['id']), + reverse=True + ) + + # return first node in sorted nodes + return sorted_nodes[0]['data']['id'] + + + + + def get_edge_by_target(self, target: str=None) -> dict: + """ + Retrieves the self.flowrun.edge[] that matched the + passed 'target' id + + Args: + 'target': str + } + + Returns: + 'index': str, + 'edge': dict + } + """ + + # find target + index = 0 + for e in self.flowrun.edges: + if e['target'] == target: + return { + 'index': index, + 'edge': e + } + index+=1 + return {'index': None, 'edge': None} + + + + + def get_edges_by_source(self, source: str=None) -> dict: + """ + Retrieves the self.flowrun.edges[] that matched the + passed 'source' id + + Args: + 'source': str + + Returns: [{ + 'index': str, + 'edge': dict + },] + """ + + # find + index = 0 + edges = [] + for e in self.flowrun.edges: + if e['source'] == source: + edges.append({ + 'index': index, + 'edge': e + }) + index+=1 + return edges + + + + + def get_node_by_id(self, id: str=None) -> dict: + """ + Retrieves the self.flowrun.node[] that matched the + passed 'id' + + Args: + 'id': str + + Returns: + 'index': str, + 'node': dict + """ + + # find node by id + index = 0 + for n in self.flowrun.nodes: + if n['id'] == id: + return { + 'index': index, + 'node': n + } + index+=1 + return {'index': None, 'node': None} + + + + + def objects_are_complete(self, object_list: list=[]) -> bool: + """ + Iterates through the object_list of a given node + and returns True if all object.status != 'working' + + Args: + 'object_list': list + + Returns: + bool + """ + if len(object_list) == 0: + return True + for obj in object_list: + if obj['status'] == 'working': + return False + return True + + + + + def check_all_working_nodes(self, ignore_ids: list=[]) -> None: + """ + Check all objs.time_complete for each working node. + if node is `working` and all obj.time_complete + are not None: update node & edge with status.'passed' + + Args: + ignore_ids: `node.ids` to ignore + + Returns: + None + """ + # get fresh flowrun obj + flowrun = FlowRun.objects.get(id=self.flowrun_id) + + # copy flowrun.nodes & flowrun.edges + nodes = flowrun.nodes + edges = flowrun.edges + + # set index + i = 0 + + # loop through all nodes + for node in flowrun.nodes: + if node['data']['status'] == 'working' and node['id'] not in ignore_ids: + + # loop through each "working" obj + if node['data']['objects']: + + # set defaults + status = 'passed' + + for obj in node['data']['objects']: + if obj['status'] == 'working': + + # catch all objs that have no id yet (i.e. `Test` objs) + if obj['id'] is None: + status = 'working' + continue + + # get object using Alerter() + o = Alerter( + object_id=obj['id'], + task_type=node['data']['task_type'] + ).get_object() + + # check time_complete + if o is not None: + try: + if o.time_completed is None: + status = 'working' + except Exception as e: + print(e) + pass + + # update node if changed + if status != 'working': + + # update node + j = 0 + final_status = status + for obj in nodes[i]['data'].get('objects', []): + + # get current obj status + _status = nodes[i]['data']['objects'][j]['status'] + + # update obj status + nodes[i]['data']['objects'][j]['status'] = _status if _status != 'working' else 'passed' + + # update final_status if obj failed + if _status == 'failed': + final_status = 'failed' + j += 1 + + # add final node status + nodes[i]['data']['status'] = final_status + nodes[i]['data']['time_completed'] = self.build_timestamp() + + # update edge + edge = self.get_edge_by_target(nodes[i]['data']['id']) + if edge['edge']: + edges[edge['index']]['animated'] = True if final_status == 'working' else False + edges[edge['index']]['style'] = {'stroke': "#60a5fa"} if final_status == 'working' else None + + # increment + i += 1 + + # update flowrun + flowrun.nodes = nodes + flowrun.edges = edges + flowrun.save() + + return None + + + + + def finalize_node(self, index: int=None) -> None: + """ + Updates the node matching the 'index' with + 'finalized' = True, then updates self.flowrun + + Args: + 'index': int + + Returns: None + """ + + # copy and update + nodes = self.flowrun.nodes + nodes[int(index)]['data']['finalized'] = True + + # save to DB + self.flowrun.nodes = nodes + self.flowrun.save() + return None + + + + + def complete_flowrun(self, current_data: dict=None) -> None: + """ + Checks for run completion and updates final + run status. + + Args: + 'current_data': dict + + Returns: + `FlowRun` object + """ + + # defaults + status = 'passed' + nodes = self.flowrun.nodes + logs = self.flowrun.logs + + # mark current current node as finalized + # and update status + if current_data: + index = current_data['index'] + current_status = nodes[int(index)]['data']['status'] + + nodes[int(index)]['data']['finalized'] = True + nodes[int(index)]['data']['status'] = 'passed' if current_status == 'working' else current_status + + # check all nodes statuses + for node in FlowRun.objects.get(id=self.flowrun_id).nodes: + + # check for non-current 'working' nodes + if node['data']['status'] == 'working': + if current_data: + if node['id'] != current_data['node']['id']: + return self.flowrun + else: + return self.flowrun + + # check for any 'failed' nodes + if node['data']['status'] == 'failed': + status = 'failed' + + # check for current node failure + if current_data: + current_status = nodes[int(current_data['index'])]['data']['status'] + status = current_status if current_status == 'failed' else status + + # build log data + logs.append({ + 'timestamp' : self.build_timestamp(), + 'step' : current_data['node']['id'] if current_data else self.get_last_node_id(), + 'message' : ( + f'flowrun completed with status: {"✅ PASSED" if status == "passed" else "❌ FAILED"}' + ), + }) + # sort logs + logs = sorted(logs, key=lambda l: int(l['step']),) + + # update flowrun + self.flowrun.time_completed = self.build_timestamp() + self.flowrun.status = status + self.flowrun.logs = logs + self.flowrun.nodes = nodes + self.flowrun.save() + + # run alert if requested + alert_id = current_data['node']['data'].get('alert_id') if current_data else None + if alert_id: + Alerter(alert_id=alert_id, object_id=str(self.flowrun_id)).run_alert() + + # return flowrun + return self.flowrun + + + + + def run_next(self) -> None: + """ + Checks for the next step and executes + if current step has completed. + + Args: + None + + Returns: + `FlowRun` object + """ + + # cache lookup/lock + lock_key = f'flowr:run_next:{self.flowrun_id}' + pending_key = f'{lock_key}:pending' + lock_ttl = 300 # increased from 90 sec + lock_id = str(uuid.uuid4()) + + # single run per flowrun_id + if not cache.add(lock_key, lock_id, timeout=lock_ttl): + print('[FLOWRUN] no cache lock available') + cache.set(pending_key, '1', timeout=lock_ttl) + return self.flowrun + + try: + # get fresh flowrun + self.flowrun = FlowRun.objects.get(id=self.flowrun_id) + + # check if flowrun is complete + if self.flowrun.time_completed: + # return early + print('flowrun is complete') + return self.flowrun + + # get last completed node or None + current_data = self.get_current_step() + + # check if FlowRun is just starting + if current_data['node'] is None and \ + self.flowrun.nodes[0]['data']['status'] == 'queued': + + # create step_data for first step + step_data = { + 'index': 0, + 'node': self.flowrun.nodes[0] + } + + # create alert obj if needed for first job + alert_obj = { + 'parent': str(self.flowrun_id), + 'id': str(self.flowrun_id), + 'source_id': str(self.flowrun_id), + 'track_id': str(self.flowrun_id), + 'status': 'working' + } + objs = [alert_obj,] if step_data['node']['data']['task_type'] in self.alert_types else [] + + # catch empty task_type + if not step_data['node']['data']['task_type']: + self.complete_flowrun(current_data=step_data) + return self.flowrun + + # run first step + print('running first step') + self.execute_step(step_data, objs) + return self.flowrun + + # catch updates without a current_node + if current_data['node'] is None: + return self.flowrun + + # check for node conditions given not 'queued' or 'working' + if current_data['node']['data']['conditions'] and \ + (current_data['node']['data']['status'] != 'failed' or not self.flowrun.configs.get('end_on_fail')): + + # starting conditons buliding & execution + print('building conditons') + + # finialize node + self.finalize_node(index=current_data['index']) + + # set defaults + true_outcomes = [] + false_outcomes = [] + + # iterate through the objects and run conditions for each + for obj_data in current_data['node']['data'].get('objects', []): + + # get obj using Alerter + obj = Alerter( + object_id=obj_data.get('source_id', obj_data['id']), + task_type=current_data['node']['data']['task_type'] + ).get_object() + + # check if obj is Test and if status != 'incomplete' (skip if true) + if type(obj).__name__ == 'Test': + if obj.status == 'incomplete': + continue + + # build and execute conditions + conditions = Alerter( + expressions=current_data['node']['data']['conditions'] + ).build_expressions() + + print(conditions) + + # evaluate conditons + outcome = eval(f'True if ({conditions}) else False') + + # create new fake parent ID + parentID = uuid.uuid4() + + # sorting + if outcome == True: + true_outcomes.append({ + 'parent' : str(parentID), + 'id' : obj_data['id'], + 'source_id' : obj_data.get('source_id', obj_data['id']), + 'track_id' : obj_data.get('track_id', obj_data['id']), + 'status' : 'working' + }) + if outcome == False: + false_outcomes.append({ + 'parent' : str(parentID), + 'id' : obj_data['id'], + 'source_id' : obj_data.get('source_id', obj_data['id']), + 'track_id' : obj_data.get('track_id', obj_data['id']), + 'status' : 'working' + }) + + # get child edges + edges = self.get_edges_by_source(current_data['node']['id']) + children = [self.get_node_by_id(e['edge']['target']) for e in edges] + + # establish true/false child nodes + true_child = None + false_child = None + for c in children: + if c['node']['data']['start_if'] == True: + true_child = c + if c['node']['data']['start_if'] == False: + false_child = c + + # run true_child if true_outcomes exists + if len(true_outcomes) > 0: + print('RUNNING TRUE CHILD') + true_task = true_child['node']['data']['task_type'] if true_child else None + # sleeping random for DB + time.sleep(random.uniform(1, 5)) + self.execute_step( + step_data=true_child, + objects=true_outcomes if true_task in self.alert_types else [] + ) + + # run false_child if false_outcomes exists + if len(false_outcomes) > 0: + print('RUNNING FALSE CHILD') + false_task = false_child['node']['data']['task_type'] if false_child else None + # sleeping random for DB + time.sleep(random.uniform(1, 5)) + self.execute_step( + step_data=false_child, + objects=false_outcomes if false_task in self.alert_types else [] + ) + + # ending section + return self.flowrun + + # get and execute next step if current_node status is 'passed' + if current_data['node']['data']['status'] == 'passed': + + # finialize node + self.finalize_node(index=current_data['index']) + + # get child edges + edges = self.get_edges_by_source(current_data['node']['id']) + children = [self.get_node_by_id(e['edge']['target']) for e in edges] + + # children length should be <= 1 since + # current_node.conditions == None + if len(children) == 1: + if children[0] is not None: + next_step = children[0] + if next_step['node']['data']['task_type']: + print('running next step after "PASSED" non-conditional step') + + _objs = current_data['node']['data'].get('objects', []) + parent = self.get_node_by_id(current_data['node']['data'].get('parentId')) + source_task_type = current_data['node']['data'].get('task_type') + if parent and parent.get('node'): + parent_task_type = ((parent.get('node') or {}).get('data') or {}).get('task_type') + if parent_task_type: + source_task_type = parent_task_type + objs = [] + res = [] + + # set `objs` to previous object data + if next_step['node']['data']['task_type'] in self.alert_types: + objs = _objs + + # set `res` using object data from previous step + if len(_objs) > 0: + + site_types = ['caserun', 'report'] + page_types = ['test', 'scan'] + + for obj_data in _objs: + + # get obj using Alerter + obj = Alerter( + object_id=obj_data.get('source_id', obj_data['id']), + task_type=source_task_type + ).get_object() + + if obj is None: + continue + + # get obj type + obj_type = type(obj).__name__.lower() + + # adding site as resource + if obj_type in site_types: + res = [{ + "id" : str(self.flowrun.site.id), + "str" : self.flowrun.site.site_url, + "type" : "site" + }] + + # ending loop early if site + break + + # adding associated pages as resources + elif obj_type in page_types: + res.append({ + "id" : str(obj.page.id), + "str" : obj.page.page_url, + "type" : "page" + }) + + self.execute_step(next_step, objs, res) + return self.flowrun + + # if no children, end flowrun and update logs + self.complete_flowrun(current_data=current_data) + + # return flowrun + return self.flowrun + + # mark flowrun as `complete` and `failed` if + # current_node status is 'failed' & 'end_on_fail' is True + if current_data['node']['data']['status'] == 'failed': + + # finialize node + self.finalize_node(index=current_data['index']) + + # end flowrun if requested + if self.flowrun.configs.get('end_on_fail', True): + + print('--- ending run early due to failure ---') + self.complete_flowrun(current_data=current_data) + + # return flowrun + return self.flowrun + + # get child edges + edges = self.get_edges_by_source(current_data['node']['id']) + children = [self.get_node_by_id(e['edge']['target']) for e in edges] + + # children length should be <= 1 since + # current_node.conditions == None + if len(children) == 1: + if children[0] is not None: + next_step = children[0] + # check for data in next_step + if next_step['node']['data']['task_type']: + print('running next step after "FAILED" non-conditional step') + objs = [] + if next_step['node']['data']['task_type'] in self.alert_types: + objs = current_data['node']['data'].get('objects', []) + self.execute_step(next_step, objs) + return self.flowrun + + # if no children, end flowrun as 'failed' and update logs + self.complete_flowrun(current_data=current_data) + + # return flowrun + return self.flowrun + + # log any exception + except Exception as e: + print(f'[FLOWRUN Error]: {e}') + + # handle cache cleanup + finally: + + # rm local run cache + print('[FLOWRUN] removing cache lock') + cache.delete(lock_key) + + # rm local pending lock and run + if cache.get(pending_key): + print('[FLOWRUN] running pending follow-up') + cache.delete(pending_key) + Flowr(flowrun_id=str(self.flowrun_id)).run_next() + + + + + def execute_step( + self, + step_data : dict=None, + objects : list=None, + resources : list=None, + ) -> None: + """ + Executes the `step` with associated job. + + Args: + 'step_data' : dict, + 'objects' : list, + 'resources' : list + + Returns: + None + """ + + if step_data is None: + print('no step_data provided - attempting to end run...') + self.complete_flowrun(current_data=step_data) + return + + if not step_data['node']['data']['task_type']: + print('no task_type provided - attempting to end run...') + self.complete_flowrun(current_data=step_data) + return + + # get step/node data & task_type + node_data = step_data['node']['data'] + task_type = node_data['task_type'] + node_index = step_data['index'] + message = ( + f'starting job ID: {node_data["id"]} ' + + f'| job type is [ {task_type.upper()} ]' + ) + + # update self.flowrun logs, nodes, & edges + self.flowrun = FlowRun.objects.get(id=self.flowrun_id) + nodes = self.flowrun.nodes + edges = self.flowrun.edges + logs = self.flowrun.logs + + # update current node + nodes[step_data['index']]['data']['status'] = 'working' + nodes[step_data['index']]['data']['time_started'] = self.build_timestamp() + + # update node objects only if task_type is not 'issue' or 'report' + nodes[step_data['index']]['data']['objects'] = objects if (task_type != 'issue' and task_type != 'report') else [] + + # update current edge if not first step + if step_data['index'] != 0: + edge_index = self.get_edge_by_target(target=node_data['id'])['index'] + edges[edge_index]['animated'] = True + edges[edge_index]['style'] = {'stroke': "#60a5fa"} + + # update current logs + logs.append({ + 'timestamp' : self.build_timestamp(), + 'message' : message, + 'step' : node_data['id'] + }) + + # sort logs + logs = sorted(logs, key=lambda l: int(l['step']),) + + # save updates + self.flowrun.nodes = nodes + self.flowrun.edges = edges + self.flowrun.logs = logs + self.flowrun.save() + + # build common data + scope = 'account' + configs = node_data['configs'] + flowrun_id = str(self.flowrun.id) + account_id = str(self.flowrun.account.id) + types = node_data.get('type') + resources = [{ + 'str' : self.flowrun.site.site_url, + 'id' : str(self.flowrun.site.id), + 'type' : 'site' + },] if not resources else resources + + # create new scan + if task_type == 'scan': + create_scan_bg.delay( + scope = scope, + resources = resources, + account_id = account_id, + type = types, + configs = configs, + flowrun_id = flowrun_id, + node_index = node_index + ) + + # create new test + if task_type == 'test': + create_test_bg.delay( + scope = scope, + resources = resources, + account_id = account_id, + type = types, + configs = configs, + threshold = node_data['threshold'], + flowrun_id = flowrun_id, + node_index = node_index + ) + + # create new caserun + if task_type == 'case': + create_caserun_bg.delay( + scope = scope, + resources = resources, + account_id = account_id, + case_id = node_data['case_id'], + updates = node_data['updates'], + configs = configs, + flowrun_id = flowrun_id, + node_index = node_index + ) + + # create new issue + if task_type == 'issue': + create_issue_bg.delay( + account_id = account_id, + objects = objects, + title = node_data['title'], + details = node_data['details'], + generate = node_data['generate'], + flowrun_id = flowrun_id, + node_index = node_index + ) + + # create new report + if task_type == 'report': + create_report_bg.delay( + scope = scope, + resources = resources, + account_id = account_id, + configs = configs, + flowrun_id = flowrun_id, + node_index = node_index + ) + + # send phone notification + if task_type == 'phone': + send_phone_bg.delay( + account_id = account_id, + objects = objects, + phone_number = node_data['phone_number'], + body = node_data['message'], + flowrun_id = flowrun_id, + node_index = node_index + ) + + # send slack notification + if task_type == 'slack': + send_slack_bg.delay( + account_id = account_id, + objects = objects, + body = node_data['message'], + flowrun_id = flowrun_id, + node_index = node_index + ) + + # send email notification + if task_type == 'email': + send_email_bg.delay( + account_id = account_id, + objects = objects, + message_obj = { + 'plain_text' : True, + 'email' : node_data['email'], + 'subject' : node_data['subject'], + 'content' : node_data['message'] + }, + flowrun_id = flowrun_id, + node_index = node_index + ) + + # send webhook notification + if task_type == 'webhook': + send_webhook_bg.delay( + account_id = account_id, + objects = objects, + request_type = node_data['request_type'], + url = node_data['uri'], + headers = node_data['headers'], + payload = node_data['payload'], + flowrun_id = flowrun_id, + node_index = node_index + ) + + # check all objs.time_complete for each "working" node. + # if node is `working` and all obj.time_complete + # are not None: update node with status.'passed' + self.check_all_working_nodes(ignore_ids=[node_data['id']]) + + # returning + return None + + diff --git a/app/api/utils/image.py b/app/api/utils/image.py deleted file mode 100644 index a6f88768..00000000 --- a/app/api/utils/image.py +++ /dev/null @@ -1,1163 +0,0 @@ -from .driver_s import driver_init, driver_wait, quit_driver -from .driver_p import driver_init as driver_init_p -from selenium import webdriver -from ..models import Site, Scan, Test, Mask -from selenium.webdriver.chrome.options import Options -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder -from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc -from scanerr import settings -from PIL import Image as I, ImageChops, ImageStat -from pyppeteer import launch -from datetime import datetime -from asgiref.sync import sync_to_async -import time, os, sys, json, uuid, boto3, \ - statistics, shutil, numpy, cv2 - - - - - -class Image(): - """ - High level Image handler used to compare screenshots of - a website and retrieve single one-page screenshots. - Also known as VRT or Visual Regression Testing. - Contains five methods scan(), scan_p(), test(), - screenshot(), and screenshot_p(). The _p appendage - denotes using Puppeteer as the webdriver: - - def scan(site, driver=None) -> grabs multiple - screenshots of the website and uploads - them to s3. - - - def test(test=) -> compares each - screenshot in the two scans and records - a score out of 100% - - - def screeshot(site, driver=None) -> grabs single - screenshot of the site and uploads it to s3 - - """ - - - def __init__(self): - - # scripts - self.set_jquery = ( - """ - var jq = document.createElement('script'); - jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; - document.getElementsByTagName('head')[0].appendChild(jq); - """ - ) - - - self.mask_function = ( - """ - (function($){ - $.fn.overlayMask = function (action) { - var mask = this.find('.overlay-mask'); - - // Create the required mask - - if (!mask.length) { - this.css({ - position: 'relative' - }); - mask = $('
'); - mask.css({ - position: 'absolute', - width: '100%', - height: '100%', - color: 'green', - backgroundColor: 'green', - top: '0px', - left: '0px', - zIndex: 100, - }).appendTo(this); - } - - // Act based on params - - if (!action || action === 'show') { - mask.show(); - } else if (action === 'hide') { - mask.hide(); - } - - return this; - }; - })(jQuery) - - """ - ) - - - - - def check_timeout(self, timeout, start_time): - """ - Checks to see if the current time exceedes the alotted timeout. - - returns -> True if timeout exceeded - """ - - current = datetime.now() - diff = current - start_time - if diff.total_seconds() >= timeout: - print('exceeded timeout') - return True - else: - return False - - - - - def scan(self, site, configs, driver=None,): - """ - Grabs multiple screenshots of the website and uploads - them to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # initialize driver if not passed as param - driver_present = True - if not driver: - driver = driver_init() - driver_present = False - - - # request site_url - driver.get(site.site_url) - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # inserting video pausing scripts - try: - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannnot pause videos') - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - active_masks = Mask.objects.filter(active=True) - if len(active_masks) != 0: - for mask in active_masks: - try: - driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # scroll one frame at a time and capture screenshot - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - # driver.execute_script("window.scrollBy(0, window.innerHeight);") - driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - # get screenshot - driver.save_screenshot(f'{pic_id}.png') - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site.id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "index": index, - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - index += 1 - - else: - bottom = True - - if not driver_present: - quit_driver(driver) - - return image_array - - - - - - - def _scan(self, site, configs, driver=None,): - """ - Grabs multiple screenshots of the website and uploads - them to s3 as one package. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # initialize driver if not passed as param - driver_present = True - if not driver: - driver = driver_init() - driver_present = False - - - # request site_url - driver.get(site.site_url) - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # inserting video pausing scripts - try: - driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());") - except: - print('cannnot pause videos') - - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - active_masks = Mask.objects.filter(active=True) - if len(active_masks) != 0: - for mask in active_masks: - try: - driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # vertically concats two images - def add_images(im1, im2): - im1 = I.open(im1) - im2 = I.open(im2) - new_img = I.new('RGB', (im1.width, im1.height + im2.height)) - new_img.paste(im1, (0, 0)) - new_img.paste(im2, (0, im1.height)) - return new_img - - - # scroll one frame at a time and capture screenshot - final_img = None - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - # driver.execute_script("window.scrollBy(0, window.innerHeight);") - driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # waiting for network requests to resolve - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - # get screenshot - driver.save_screenshot(f'{pic_id}.png') - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - - # adding new image to bottom of existing image (if not index = 0) - pic_id_2 = uuid.uuid4() - if index != 0 and final_img is not None: - add_images(final_img, image).save(f'{pic_id_2}.png') - os.remove(final_img) - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - else: - I.open(image).save(f'{pic_id_2}.png') - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - - # remove local copy - os.remove(image) - - index += 1 - - else: - bottom = True - - - remote_path = f'static/sites/{site.id}/{pic_id_2}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(final_img, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - - - # create image obj and add to list - img_obj = { - "index": 0, - "id": str(pic_id_2), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - # remove local copy - os.remove(final_img) - - if not driver_present: - quit_driver(driver) - - return image_array - - - - - - - - async def scan_p(self, site, configs): - """ - Using Puppeteer, grabs multiple screenshots of the website and uploads - them to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) - page = await driver.newPage() - - sizes = configs.get('window_size', '1920,1080').split(',') - is_mobile = False - if configs.get('device') == 'mobile': - is_mobile = True - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs.get('max_wait_time', 30)*1000 - } - - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent - } - - if configs.get('device') == 'mobile': - await page.emulate(emulate_options) - else: - await page.setViewport(viewport) - - # requesting site url - await page.goto(site.site_url, page_options) - - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # pausing videos - try: - videos = await page.querySelectorAll('video') - for vid in videos: - await page.evaluate('(vid) => vid.pause()', vid) - except Exception as e: - print(e) - - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - @sync_to_async - def get_active_global_masks(): - masks = Mask.objects.filter(active=True) - active_masks = [] - if len(masks) > 0: - for mask in masks: - active_masks.append(mask.id) - return active_masks - - active_masks = await get_active_global_masks() - - for mask in active_masks: - try: - await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # scroll one frame at a time and capture screenshot - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # interact with and wait for page to load - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - time.sleep(configs.get('min_wait_time', 10)) - - - # get screenshot - await page.screenshot({'path': f'{pic_id}.png'}) - - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site.id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "index": index, - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - index += 1 - - else: - bottom = True - - - await driver.close() - - return image_array - - - - - - - - - async def _scan_p(self, site, configs): - """ - Using Puppeteer, grabs multiple screenshots of the website and uploads - them to s3 as a single image. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) - page = await driver.newPage() - - sizes = configs.get('window_size', '1920,1080').split(',') - is_mobile = False - if configs.get('device') == 'mobile': - is_mobile = True - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs.get('max_wait_time', 30)*1000 - } - - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent - } - - if configs.get('device') == 'mobile': - await page.emulate(emulate_options) - else: - await page.setViewport(viewport) - - # requesting site url - await page.goto(site.site_url, page_options) - - if configs.get('disable_animations') == True: - # inserting animation pausing script - try: - await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);") - except: - print('cannot pause animations') - - # pausing videos - try: - videos = await page.querySelectorAll('video') - for vid in videos: - await page.evaluate('(vid) => vid.pause()', vid) - except Exception as e: - print(e) - - # mask all listed ids - if configs.get('mask_ids') is not None and configs.get('mask_ids') != '': - ids = configs.get('mask_ids').split(',') - for id in ids: - try: - await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via id provided') - - - # mask all Global mask ids that are active - @sync_to_async - def get_active_global_masks(): - masks = Mask.objects.filter(active=True) - active_masks = [] - for mask in masks: - active_masks.append(mask.id) - return active_masks - - active_masks = await get_active_global_masks() - - for mask in active_masks: - try: - await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';") - print('masked an element') - except: - print('cannot find element via global mask id provided') - - - # vertically concats two images - @sync_to_async - def add_images(im1, im2): - im1 = I.open(im1) - im2 = I.open(im2) - new_img = I.new('RGB', (im1.width, im1.height + im2.height)) - new_img.paste(im1, (0, 0)) - new_img.paste(im2, (0, im1.height)) - return new_img - - - # scroll one frame at a time and capture screenshot - final_img = None - image_array = [] - index = 0 - last_height = -1 - bottom = False - start_time = datetime.now() - while not bottom: - - # checking if maxed out time - if self.check_timeout(configs.get('timeout', 300), start_time): - break - - # scroll single frame - if index != 0: - await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);") - time.sleep(int(configs.get('min_wait_time', 10))) - - # get current position and compare to previous - new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight") - height_diff = new_height - last_height - if height_diff > 20: - last_height = new_height - pic_id = uuid.uuid4() - - # interact with and wait for page to load - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - time.sleep(configs.get('min_wait_time', 10)) - - - # get screenshot - await page.screenshot({'path': f'{pic_id}.png'}) - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - - # adding new image to bottom of existing image (if not index = 0) - pic_id_2 = uuid.uuid4() - if index != 0 and final_img is not None: - new_img = await add_images(final_img, image) - new_img.save(f'{pic_id_2}.png') - os.remove(final_img) - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - else: - I.open(image).save(f'{pic_id_2}.png') - final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') - - # remove local copy - os.remove(image) - - index += 1 - - else: - bottom = True - - - remote_path = f'static/sites/{site.id}/{pic_id_2}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(final_img, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - - - # create image obj and add to list - img_obj = { - "index": 0, - "id": str(pic_id_2), - "url": image_url, - "path": remote_path, - } - - image_array.append(img_obj) - - # remove local copy - os.remove(final_img) - - - await driver.close() - - return image_array - - - - - - - - - - - def test(self, test, index=None): - """ - Compares each screenshot between the two scans and records - a score out of 100%. - - Compairsons used : - - Structral Similarity Index (ssim) - - PIL ImageChop Differences, Ratio - - cv2 ORB Brute-force Matcher, Ratio - - - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # setup temp dirs - if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')): - os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}')) - - # temp root - temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}') - - # loop through and download each img in scan and compare it. - pre_scan_images = test.pre_scan.images - img_test_results = [] - scores = [] - i = 0 - - if index is not None: - pre_scan_images = [test.pre_scan.images[index]] - i = index - - for pre_img_obj in pre_scan_images: - - # getting pre_scan image - pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png') - with open(pre_img_path, 'wb') as data: - s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data) - - # open with PIL Image library - pre_img = I.open(pre_img_path) - # convert to array - pre_img_array = numpy.array(pre_img) - - # getting post_scan image - try: - post_img_obj = test.post_scan.images[i] - except: - post_img_obj = None - - if post_img_obj is not None: - post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png') - with open(post_img_path, 'wb') as data: - s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data) - - # open with PIL Image library - post_img = I.open(post_img_path) - # convert to array - post_img_array = numpy.array(post_img) - - - # test images with PIL - def pil_score(pre_img, post_img): - try: - if (pre_img.mode != post_img.mode) \ - or (pre_img.size != post_img.size) \ - or (pre_img.getbands() != post_img.getbands()): - raise Exception('images are not comparable') - - # Generate diff image in memory. - diff_img = ImageChops.difference(pre_img, post_img) - - # Calculate difference as a ratio. - stat = ImageStat.Stat(diff_img) - diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 - pil_img_score = (100 - diff_ratio) - # print(f'PIL score -> {pil_img_score}') - return pil_img_score - - except Exception as e: - print(e) - - - # test with cv2 - def cv2_score(pre_img_array, post_img_array): - try: - orb = cv2.ORB_create() - - # detect keypoints and descriptors - kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) - kp_b, desc_b = orb.detectAndCompute(post_img_array, None) - - # define the bruteforce matcher object - bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) - - # perform matches. - matches = bf.match(desc_a, desc_b) - - # Look for similar regions with distance < 20. (from 0 to 100) - similar_regions = [i for i in matches if i.distance < 20] - if len(matches) == 0: - cv2_img_score = 100 - else: - cv2_img_score = (len(similar_regions) / len(matches)) * 100 - # print(f'cv2 -> {cv2_img_score}') - - return cv2_img_score - - except Exception as e: - print(e) - - - - # test images - try: - img_score_tupple = ssim(pre_img_array, post_img_array) - img_score_list = list(img_score_tupple) - ssim_img_score = statistics.fmean(img_score_list) * 100 - # print(f'ssim -> {ssim_img_score}') - - pil_img_score = pil_score(pre_img, post_img) - # print(f'pil -> {pil_img_score}') - - cv2_img_score = cv2_score(pre_img_array, post_img_array) - # print(f'cv2 -> {cv2_img_score}') - - img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 - # print(f'img_score ==> {img_score}') - - except Exception as e: - print(e) - img_score = None - - # create img test obj and add to array - img_test_obj = { - "index": i, - "pre_img": pre_img_obj, - "post_img": post_img_obj, - "score": img_score, - } - - img_test_results.append(img_test_obj) - scores.append(img_score) - - # remove local copies - if post_img_obj is not None: - try: - os.remove(post_img_path) - except Exception as e: - print(e) - try: - os.remove(pre_img_path) - except Exception as e: - print(e) - - i += 1 - - # remove temp dir - shutil.rmtree(temp_root) - - # averaging scores and storing in images_delta obj - try: - avg_score = statistics.fmean(scores) - except: - avg_score = None - - images_delta = { - "average_score": avg_score, - "images": img_test_results, - } - - return images_delta - - - - - - - - def screenshot(self, site=None, url=None, configs=None, driver=None): - """ - Grabs single screenshot of the website and uploads - it to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - if not configs: - configs = { - "interval": 5, - "window_size": "1920,1080", - "max_wait_time": 60, - "min_wait_time": 10, - "device": "desktop" - } - - # initialize driver if not passed as param - if not driver: - driver = driver_init(window_size=configs.get('window_size', '1920,1080'), device=configs.get('device')) - - - # get or create site data - if site is None: - site_id = uuid.uuid4() - site_url = url - else: - site_id = site.id - site_url = site.site_url - - # request site_url - driver.get(site_url) - - - # wait for site to fully load - driver_wait( - driver=driver, - interval=int(configs.get('interval', 5)), - min_wait_time=int(configs.get('min_wait_time', 10)), - max_wait_time=int(configs.get('max_wait_time', 30)), - ) - - # grab screenshot - pic_id = uuid.uuid4() - driver.save_screenshot(f'{pic_id}.png') - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site_id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - # quit driver - quit_driver(driver) - - return img_obj - - - - async def screenshot_p(self, site=None, url=None, configs=None): - """ - Using Puppeteer, grabs single screenshot of the website and uploads - it to s3. - """ - - # setup boto3 configurations - s3 = boto3.client( - 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - if not configs: - configs = { - "interval": 5, - "driver": "puppeteer", - "device": "desktop", - "window_size": "1920,1080", - "max_wait_time": 60, - "min_wait_time": 10 - } - - driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30)) - page = await driver.newPage() - - sizes = configs.get('window_size', '1920,1080').split(',') - is_mobile = False - if configs.get('device') == 'mobile': - is_mobile = True - - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': configs.get('max_wait_time', 30)*1000 - } - - viewport = { - 'width': int(sizes[0]), - 'height': int(sizes[1]), - 'isMobile': is_mobile, - } - - userAgent = ( - "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \ - (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36" - ) - - emulate_options = { - 'viewport': viewport, - 'userAgent': userAgent - } - - if configs.get('device') == 'mobile': - await page.emulate(emulate_options) - else: - await page.setViewport(viewport) - - # get or create site data - if site is None: - site_id = uuid.uuid4() - site_url = url - else: - site_id = site.id - site_url = site.site_url - - # request site_url - await page.goto(site_url, page_options) - - # interact with and wait for page to load - await page.mouse.move(0, 0) - await page.mouse.move(0, 100) - time.sleep(configs.get('min_wait_time', 10)) - - # get screenshot - pic_id = uuid.uuid4() - await page.screenshot({'path': f'{pic_id}.png'}) - await driver.close() - image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') - remote_path = f'static/sites/{site_id}/{pic_id}.png' - root_path = settings.AWS_S3_URL_PATH - image_url = f'{root_path}/{remote_path}' - - # upload to s3 - with open(image, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} - ) - # remove local copy - os.remove(image) - - # create image obj and add to list - img_obj = { - "id": str(pic_id), - "url": image_url, - "path": remote_path, - } - - return img_obj \ No newline at end of file diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py new file mode 100644 index 00000000..b7db8e9a --- /dev/null +++ b/app/api/utils/imager.py @@ -0,0 +1,1133 @@ +from .driver import driver_init, driver_wait, quit_driver +from ..models import Mask +from skimage.metrics import structural_similarity +from cursion import settings +from PIL import Image as I, ImageChops, ImageStat +from datetime import datetime +from openai import OpenAI +from pydantic import BaseModel +from .meter import meter_account +import time, os, uuid, boto3, \ + base64, shutil, numpy, cv2, requests + + + + + + +class Imager(): + """ + High level Image handler used to compare screenshots + of a website. + + Also known as VRT or Visual Regression Testing. + Contains three methods scan_vrt(), test_vrt(), + & caserun_vrt(): + + def scan_vrt(driver=None) -> using selenium + grabs screenshots of the website + and uploads them to s3. + + def test_vrt() -> compares each + screenshot in the Test + + def caserun_vrt() -> compares the + screenshot in each step of a CaseRun + """ + + + + + def __init__( + self, + scan : object=None, + test : object=None, + caserun : object=None + ): + + # primary objects + self.scan = scan + self.test = test + self.caserun = caserun + + # main image_array for scans + self.image_array = [] + + # setup boto3 configurations + self.s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # scripts + self.pause_video_script = ( + """ + document.querySelectorAll('video').forEach(vid => vid.pause()); + document.querySelectorAll('video').forEach(vid => vid.currentTime=0); + """ + ) + self.set_jquery = ( + """ + var jq = document.createElement('script'); + jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"; + document.getElementsByTagName('head')[0].appendChild(jq); + """ + ) + self.pause_animations_script = ( + """ + const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag'); + const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}'); + styleElement.appendChild(styleTagCSSes); + document.head.appendChild(styleElement); + """ + ) + self.pause_stretch = ( + """ + (() => { + const elements = [document.documentElement, document.body]; + elements.forEach(el => {el.style.backgroundAttachment = 'fixed'; el.style.backgroundRepeat = 'no-repeat'; el.style.backgroundSize = 'auto';}); + const fullHeightDivs = document.querySelectorAll('[style*="background"], [class*="background"]'); + fullHeightDivs.forEach(el => {const style = window.getComputedStyle(el);if (style.backgroundImage!=='none') {el.style.backgroundAttachment='fixed'; el.style.backgroundSize='auto';} + }); + })(); + """ + ) + + + + + def check_timeout(self, timeout: int, start_time: str) -> bool: + """ + Checks to see if the current time exceedes the alotted timeout. + + Returns: True if timeout exceeded + """ + current = datetime.now() + diff = current - start_time + if diff.total_seconds() >= int(timeout): + print('exceeded timeout') + return True + else: + return False + + + + + def add_images(self, im1: object, im2: object) -> object: + """ + Joins img1 and im2 vertically and saves as "new_img" + + Returns: new_img + """ + im1 = I.open(im1) + im2 = I.open(im2) + new_img = I.new('RGB', (im1.width, im1.height + im2.height)) + new_img.paste(im1, (0, 0)) + new_img.paste(im2, (0, im1.height)) + return new_img + + + + + def save_image(self, pic_id: str, image: str) -> None: + """ + Upload image to s3, save info as image_obj, + add image_obj to image_array, & remove image file + + Args: + pic_id: str, + image: str + + Returns: + None + """ + remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + # create image obj and add to list + img_obj = { + "index": 0, + "id": str(pic_id), + "url": image_url, + "path": remote_path, + } + self.image_array.append(img_obj) + + print(f'adding {img_obj["url"]} to image_array') + + # remove local copy + os.remove(image) + + return None + + + + + def save_images( + self, + pre_img_id: id=None, + post_img_id: id=None, + index: int=0 + ) -> dict: + """ + Saves two images to test.id path in S3 bucket + + Args: + pre_img_id : uuid, + post_img_id : uuid, + index : int + + Returns: + img_objs : list + """ + + # build paths based test + if self.test: + remote_root = f'static/sites/{self.test.page.site.id}/{self.test.page.id}/{self.test.id}/' + temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.test.id}') + + # build paths based caserun + if self.caserun: + remote_root = f'static/caseruns/{self.caserun.id}/' + temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}') + + image_ids = [pre_img_id, post_img_id] + img_objs = [] + for img_id in image_ids: + image = os.path.join(temp_root, f'{img_id}.png') + remote_path = f'{remote_root}{img_id}.png' + root_path = settings.AWS_S3_URL_PATH + image_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(image, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"} + ) + + # building img obj + img_objs.append({ + "id": str(img_id), + "url": image_url, + "path": remote_path, + "index": index, + }) + + return img_objs + + + + + def download_image( + self, + url: str=None, + temp_root: str=None + ) -> dict: + """ + Parses image info and downloads image to local temp_root + + Args: + 'url' : str, image url, + 'temp_root' : str, local temp dir + + Returns: + 'name' : str, image name, + 'id' : str, image id, + 'remote_path' : str, remote path, + 'local_path' : str, local path + """ + image_name = url.split('/')[-1] + image_id = image_name.split('.')[0] + remote_path = f'static{url.split('static')[1]}' + local_path = os.path.join(temp_root, image_name) + + with open(local_path, 'wb') as data: + self.s3.download_fileobj( + settings.AWS_STORAGE_BUCKET_NAME, + remote_path, + data + ) + + # return data + return { + 'name': image_name, + 'id': image_id, + 'remote_path': remote_path, + 'local_path': local_path + } + + + + + def highlight_diffs( + self, + temp_root: str=None, + pre_img_path: str=None, + post_img_path: str=None, + index: int=None + ) -> dict: + """ + Runs SSIM comparision and highlights + differences between two passed images + + Args: + temp_root : str, + pre_img_path : str, + post_img_path : str, + index : int + + Returns: + img_objs : dict, + ssim_score : float + """ + # Load the images + image1 = cv2.imread(pre_img_path) + image2 = cv2.imread(post_img_path) + + # Convert the images to grayscale + gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY) + gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY) + + # Compute the SSIM map + (ssim_score, diff) = structural_similarity(gray1, gray2, full=True) + + # Highlight the differences + diff = (diff * 255).astype("uint8") + + # Threshold the difference map + _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU) + + # Find contours of the differences + contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + # Draw rectangles around the differences + for contour in contours: + (x, y, w, h) = cv2.boundingRect(contour) + cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2) + cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2) + + # Save the output images + img_1_id = uuid.uuid4() + img_2_id = uuid.uuid4() + cv2.imwrite(temp_root + f"/{img_1_id}.png", image1) + cv2.imwrite(temp_root + f"/{img_2_id}.png", image2) + img_objs = self.save_images(img_1_id, img_2_id, index) + + data = { + "img_objs": img_objs, + "ssim_score": ssim_score + } + + return data + + + + + def pil_score( + self, + pre_img: object=None, + post_img: object=None + ) -> float: + """ + Runs pixel ratio comparison on the two + passed images and returns a score. + + Args: + pre_img : object, + post_img : object, + + Returns: + pil_img_score float + """ + try: + if (pre_img.mode != post_img.mode) \ + or (pre_img.size != post_img.size) \ + or (pre_img.getbands() != post_img.getbands()): + raise Exception('images are not comparable') + + # Generate diff image in memory. + diff_img = ImageChops.difference(pre_img, post_img) + + # Calculate difference as a ratio. + stat = ImageStat.Stat(diff_img) + diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100 + pil_img_score = (100 - diff_ratio) + # print(f'PIL score -> {pil_img_score}') + return pil_img_score + + except Exception as e: + print(e) + + + + + def cv2_score( + self, + pre_img: object=None, + post_img: object=None + ) -> float: + """ + Runs cv2 ORB Brute-force comparison on the two + passed images and returns a score. + + Args: + pre_img : object, + post_img : object, + + Returns: + cv2_img_score float + """ + try: + orb = cv2.ORB_create() + + # convert to array + pre_img_array = numpy.array(pre_img) + post_img_array = numpy.array(post_img) + + # detect keypoints and descriptors + kp_a, desc_a = orb.detectAndCompute(pre_img_array, None) + kp_b, desc_b = orb.detectAndCompute(post_img_array, None) + + # define the bruteforce matcher object + bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True) + + # perform matches. + matches = bf.match(desc_a, desc_b) + + # Look for similar regions with distance < 20. (from 0 to 100) + similar_regions = [i for i in matches if i.distance < 20] + if len(matches) == 0: + cv2_img_score = 100 + else: + cv2_img_score = (len(similar_regions) / len(matches)) * 100 + # print(f'cv2 -> {cv2_img_score}') + + return cv2_img_score + + except Exception as e: + print(e) + + + + + def ai_compare( + self, + pre_img_url: str=None, + post_img_url: str=None, + score: float=None, + highlighted: bool=False + ) -> dict: + """ + Using OpenAI, compares the two images and + provides a summary and boolean for 'broken' + + Args: + pre_img_url : str, + post_img_url : str, + score : float + + Returns: + 'summary': str, + 'broken': bool + """ + + # define output as object (JSON) + class Result(BaseModel): + summary: str + broken: bool + + # init client + gpt_client = OpenAI(api_key=settings.GPT_API_KEY,) + + + marked_up_images = str( + "I've added green boxes arround the areas that have changed between the two images. \ + The green boxes may not be present if there are no changes. \ + Omit any reference to the green boxes in your response." + ) + + # send request + response = gpt_client.beta.chat.completions.parse( + model="gpt-4o-mini", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": f"Attached are two screenshots of the same website. \ + I've calculated the Visual Regression SSIM score to be {score}% similar. \ + Please perform a Visual Regression Analysis of the two images. \ + {marked_up_images if highlighted else ''} \ + Respond with a few sentance summary about what has changed. \ + Look for changes in pictures, buttons, forms, vertial shifts, etc. \ + Respond also with a boolean that is TRUE if the page should be considered broken. \ + Consider any emerging portions that appear to be unrendered HTML a breaking change. \ + If the same text is present in both images, then DO NOT consider it a 'breaking change'. \ + Only consider 'breaking changes' on the second image. \ + DO NOT consider new or altered text to be a 'breaking change'. \ + DO NOT consider text changes within images or pictures on the webpage. \ + DO NOT consider minor shifts (only a few pixels) to be a 'breaking change'. \ + Ignore portions that appear to be advertizements. \ + Please be somewhat strict with the analysis. \ + Format response as a JSON object with 'summary': , 'broken': " + }, + { + "type": "image_url", + "image_url": { + "url": pre_img_url, + }, + }, + { + "type": "image_url", + "image_url": { + "url": post_img_url, + }, + }, + ], + } + ], + response_format=Result + ) + + try: + result = response.choices[0].message.parsed + result = { + 'summary': result.summary, + 'broken': result.broken + } + except: + result = { + 'summary': None, + 'broken': None + } + + # meter account if necessary + if self.test.page.account.type == 'cloud' and self.test.page.account.cust_id: + meter_account(str(self.test.page.account.id), 2) + + print(result) + return result + + + + + def resize_window( + self, + driver: object=None, + sizes: list=[] + ) -> object: + """ + Tries to resize the window length by scrolling to + the bottom of the page. + + Args: + driver: ``, + sizes: `` + + Returns: + `Driver` + """ + + # pausing image stretching + driver.execute_script(self.pause_stretch) + + # get scroll_height, client_height & set window_size + scroll_height = driver.execute_script("return document.documentElement.scrollHeight;") + client_height = driver.execute_script("return document.documentElement.clientHeight;") + + # trying to match "document.body.clientHeight" + # and "document.body.scrollHeight" + # iterate 3 times or untill height_diff is less than 20 + i = 0 + while i < 4: + + # set window_size + driver.set_window_size(int(sizes[0]), (int(scroll_height))) + + # scroll down + driver.execute_script(f"window.scrollBy(0, {client_height});") + + # wait for content to load + driver_wait( + driver=driver, + interval=int(self.scan.configs.get('interval', 5)), + min_wait_time=int(self.scan.configs.get('min_wait_time', 10)), + max_wait_time=int(self.scan.configs.get('max_wait_time', 30)), + ) + + # scroll up + driver.execute_script(f"window.scrollBy(0, -{client_height});") + + # get client & new scroll height + client_height = driver.execute_script("return document.documentElement.clientHeight;") + new_scroll_height = driver.execute_script("return document.documentElement.scrollHeight;") + + # get difference between full page height and new scrolled position + height_diff = int(new_scroll_height) - int(client_height) + + # re-set window size + print(f'adding {height_diff} to full_page_height') + scroll_height += height_diff if height_diff > 0 else 0 + + # catching infinite scrolls + if (i >= 1 and height_diff >= 200) or height_diff >= 1000: + break + + # checking difference + if height_diff < 20: + break + + # increment + i += 1 + + return driver + + + + + def autoheight_screenshot( + self, + driver: object=None, + sizes: list=[], + browser: str='chrome' + ) -> object: + + """ + Captures full-page screenshots of the current page + using browser specifc functions. + + Args: + - driver, + - sizes, + - browser + + Returns: + - driver + """ + + # setting defaults + pic_id = uuid.uuid4() + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # seting window size to configs before resize + driver.set_window_size(sizes[0], sizes[1]) + + # handle chrome & edge cases + if browser in ['chrome', 'edge']: + + # get page height + metrics = driver.execute_cdp_cmd("Page.getLayoutMetrics", {}) + height = max(int(metrics["contentSize"]["height"]), int(sizes[1])) + + # set viewport + driver.set_window_size(sizes[0], height) + + # wait for content to load + driver_wait( + driver=driver, + interval=int(self.scan.configs.get('interval', 5)), + min_wait_time=int(self.scan.configs.get('min_wait_time', 10)), + max_wait_time=int(self.scan.configs.get('max_wait_time', 30)), + ) + + # capture screenshot using CDP + screenshot = driver.execute_cdp_cmd("Page.captureScreenshot", { + "format": "png", + "fromSurface": True, + "captureBeyondViewport": True, + }) + + # dcecode and save + with open(image, "wb") as f: + f.write(base64.b64decode(screenshot['data'])) + + # handle firefox cases + if browser == 'firefox': + + # execute firefox sepcific screenshot function + driver.get_full_page_screenshot_as_file(image) + + # save and upload + self.save_image(pic_id, image) + + # return driver + return driver + + + + + def scroll_and_stitch_screenshot( + self, + driver: object=None, + ) -> object: + """ + Captures full-page screenshots of the current page + using the scroll-and-stitch method + + Args: + - driver, + + Returns: + - driver + """ + # scroll one frame at a time and capture screenshot + final_img = None + index = 0 + last_height = -1 + bottom = False + start_time = datetime.now() + while not bottom: + + # checking if maxed out time + if self.check_timeout(self.scan.configs.get('timeout', 300), start_time): + break + + # scroll single frame if not first frame and not auto_height + if index != 0: + driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);") + time.sleep(int(self.scan.configs.get('min_wait_time', 10))) + + # get current position and compare to previous + new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight") + height_diff = new_height - last_height + print(f'new_height => {new_height} | height_diff => {height_diff}') + + if height_diff > 20: + last_height = new_height + pic_id = uuid.uuid4() + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(self.scan.configs.get('interval', 5)), + min_wait_time=int(self.scan.configs.get('min_wait_time', 10)), + max_wait_time=int(self.scan.configs.get('max_wait_time', 30)), + ) + + # get screenshot + driver.save_screenshot(f'{pic_id}.png') + image = os.path.join(settings.BASE_DIR, f'{pic_id}.png') + + # resizing image to remove duplicate portions + if index != 0: + img = I.open(image) + width, height = img.size + left = 0 + top = height - ((height_diff/2)) # divide by 2 for "driver.scale_factor" + right = width + botm = height + new_img = img.crop((left, top, right, botm)) + new_img.save(image, quality=100) + + # adding new image to bottom of existing image (if not index = 0) + pic_id_2 = uuid.uuid4() + if index != 0 and final_img is not None: + self.add_images(final_img, image).save(f'{pic_id_2}.png') + os.remove(final_img) + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + else: + I.open(image).save(f'{pic_id_2}.png') + final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png') + + os.remove(image) + index += 1 + + else: + bottom = True + + # saving image + self.save_image(pic_id=pic_id_2, image=final_img) + return driver + + + + + def scan_vrt( + self, + driver: object=None + ) -> list: + """ + Grabs full length screenshots of the website and uploads + them to s3. + + Args: { + 'driver': object + } + + Returns: self.image_array list + """ + + # initialize driver if not passed as param + driver_present = True + if not driver: + driver = driver_init( + browser=self.scan.configs.get('browser', 'chrome'), + window_size=self.scan.configs.get('window_size', '1920,1080'), + device=self.scan.configs.get('device', 'desktop'), + ) + driver_present = False + + # request page_url + driver.get(self.scan.page.page_url) + + # waiting for network requests to resolve + driver_wait( + driver=driver, + interval=int(self.scan.configs.get('interval', 5)), + min_wait_time=int(self.scan.configs.get('min_wait_time', 10)), + max_wait_time=int(self.scan.configs.get('max_wait_time', 30)), + ) + + # defining browser demesions + sizes = self.scan.configs.get('window_size', '1920,1080').split(',') + + # calculating and auto setting page height + if self.scan.configs.get('auto_height', True): + self.resize_window(driver, sizes) + + + if self.scan.configs.get('disable_animations') == True: + # inserting animation pausing script + try: + driver.execute_script(self.pause_animations_script) + except: + print('cannot pause animations') + + # inserting video pausing scripts + try: + driver.execute_script(self.pause_video_script) + except: + print('cannnot pause videos') + + # mask all listed ids + if self.scan.configs.get('mask_ids') is not None and self.scan.configs.get('mask_ids') != '': + ids = self.scan.configs.get('mask_ids').split(',') + for id in ids: + try: + driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via id provided') + + # mask all Global mask ids that are active + active_masks = Mask.objects.filter(active=True) + if len(active_masks) != 0: + for mask in active_masks: + try: + driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';") + print('masked an element') + except: + print('cannot find element via global mask id provided') + + # capture auto-height screenshot if requested + if self.scan.configs.get('auto_height', True): + + # capturing via browser specific functions + self.autoheight_screenshot( + driver=driver, + sizes=sizes, + browser=self.scan.configs.get('browser', 'chrome') + ) + + # capture non auto-height screenshot if requested + if not self.scan.configs.get('auto_height', True): + + # capturing via scroll-and-stitch method + self.scroll_and_stitch_screenshot( + driver=driver + ) + + # clean up + if not driver_present: + quit_driver(driver) + + # return images + return self.image_array + + + + + def test_vrt(self) -> dict: + """ + Compares each screenshot between the two scans and records + a score out of 100%. + + Compairsons used: + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + Args: + None + + Returns: + 'average_score' float(0-100), + 'images' dict + """ + + # defaults + i = 0 + img_score = None + pre_img = None + post_img = None + pre_img_diff = None + post_img_diff = None + ai_summary = None + broken = None + images_delta = { + "average_score": None, + "images": None, + } + + # setup temp dirs + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{self.test.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{self.test.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.test.id}') + + # catching user error + if self.test.pre_scan.images is None or self.test.post_scan.images is None: + shutil.rmtree(temp_root) + return images_delta + + # download images + pre_img_info = self.download_image(self.test.pre_scan.images[0].get('url'), temp_root) + post_img_info = self.download_image(self.test.post_scan.images[0].get('url'), temp_root) + + # open images with PIL Image library + pre_img = I.open(pre_img_info.get('local_path')) + post_img = I.open(post_img_info.get('local_path')) + + # check and reformat image sizes if necessary + pre_img_w, pre_img_h = pre_img.size + post_img_w, post_img_h = post_img.size + + # pre_img is longer + if pre_img_h > post_img_h: + print(f'pre_img is larger, adjusting...') + new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode) + new_pre_img.save(pre_img_info.get('local_path'), quality=100) + pre_img = I.open(pre_img_info.get('local_path')) + # post_img is longer + if post_img_h > pre_img_h: + print(f'post_img is larger, adjusting...') + new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode) + new_post_img.save(post_img_info.get('local_path'), quality=100) + post_img = I.open(post_img_info.get('local_path')) + + # test images + try: + # generating new highlighted images and score via ssim + ssim_results = self.highlight_diffs( + temp_root, + pre_img_info.get('local_path'), + post_img_info.get('local_path'), + i + ) + pre_img_diff = ssim_results['img_objs'][0] + post_img_diff = ssim_results['img_objs'][1] + + # ssim scoring + ssim_img_score = ssim_results['ssim_score'] * 100 + + # pillow scoring + pil_img_score = self.pil_score(pre_img, post_img) + + # pixel perfect scoring + cv2_img_score = self.cv2_score(pre_img, post_img) + + # weighted average + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + + # saving old images to test.id path + old_imgs = self.save_images(pre_img_info.get('id'), post_img_info.get('id'), i) + pre_img = old_imgs[0] + post_img = old_imgs[1] + + # running AI comparison + if self.test.post_scan.configs.get('ai_analysis') == True: + resp = self.ai_compare( + pre_img_url = self.test.pre_scan.images[0].get('url'), + post_img_url = self.test.post_scan.images[0].get('url'), + score = ssim_img_score, + highlighted = False + ) + ai_summary = resp.get('summary') + broken = resp.get('broken') + + except Exception as e: + print(e) + # reset values to None + img_score = None + pre_img = None + post_img = None + pre_img_diff = None + post_img_diff = None + ai_summary = None + broken = None + + + # create img test obj and add to array + img_test_obj = [{ + "index": 0, + "pre_img" : pre_img, + "post_img" : post_img, + "pre_img_diff" : pre_img_diff, + "post_img_diff" : post_img_diff, + "score" : img_score, + }] + + # remove temp dir + shutil.rmtree(temp_root) + + # formatting response + images_delta = { + "average_score" : img_score, + "images" : img_test_obj, + "ai_summary" : ai_summary, + "broken" : broken + } + + # returning response + return images_delta + + + + + def caserun_vrt( + self, + step: int=None, + type: str=None + ) -> dict: + """ + Compares the passed step.screenshot to the case.step.screenshot + and records a score out of 100%. + + Compairsons used: + - Structral Similarity Index (ssim) + - PIL ImageChop Differences, Ratio + - cv2 ORB Brute-force Matcher, Ratio + + Args: + step : int, current step to test + type : str, "action" or "assertion" + + + Returns: + 'average_score' : float(0-100), + 'images' : dict, + + """ + + # default + images_delta = { + "average_score": None, + "images": [{ + "index": step, + "pre_img": None, + "post_img": None, + "pre_img_diff": None, + "post_img_diff": None, + "score": None, + }], + } + + # setup temp dirs + if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}')): + os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}')) + + # temp root + temp_root = os.path.join(settings.BASE_DIR, f'temp/{self.caserun.id}') + + # get image urls + case_image_url = requests.get(self.caserun.case.steps['url']).json()[step][type].get('image') + caserun_image_url = self.caserun.steps[step][type].get('image') + + # catch null urls and return early + if case_image_url is None or caserun_image_url is None: + shutil.rmtree(temp_root) + return images_delta + + # download images + case_img_info = self.download_image(case_image_url, temp_root) + caserun_img_info = self.download_image(caserun_image_url, temp_root) + + # open images with PIL Image library + pre_img = I.open(case_img_info.get('local_path')) + post_img = I.open(caserun_img_info.get('local_path')) + + # test images + try: + # generating new highlighted images and score via ssim + ssim_results = self.highlight_diffs( + temp_root, + case_img_info.get('local_path'), + caserun_img_info.get('local_path'), + step + ) + pre_img_diff = ssim_results['img_objs'][0] + post_img_diff = ssim_results['img_objs'][1] + + # ssim scoring + ssim_img_score = ssim_results['ssim_score'] * 100 + + # pillow scoring + pil_img_score = self.pil_score(pre_img, post_img) + + # pixel perfect scoring + cv2_img_score = self.cv2_score(pre_img, post_img) + + # weighted average + img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8 + + # saving old images to caserun.id path + old_imgs = self.save_images(case_img_info.get('id'), caserun_img_info.get('id'), step) + pre_img = old_imgs[0] + post_img = old_imgs[1] + + except Exception as e: + print(e) + img_score = None + pre_img = None + post_img = None + pre_img_diff = None + post_img_diff = None + + # create img obj and add to array + img_obj = [{ + "index": step, + "pre_img": pre_img, + "post_img": post_img, + "pre_img_diff": pre_img_diff, + "post_img_diff": post_img_diff, + "score": img_score, + }] + + # remove temp dir + shutil.rmtree(temp_root) + + # formatting response + images_delta = { + "average_score": img_score, + "images": img_obj, + } + + # returning response + return images_delta + + + + + + diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py new file mode 100644 index 00000000..f1431416 --- /dev/null +++ b/app/api/utils/issuer.py @@ -0,0 +1,610 @@ +from ..models import * +from cursion import settings +from openai import OpenAI +from .meter import meter_account +import re, requests, tiktoken + + + + + + +class Issuer(): + """ + Generate new `Issue` for the passed 'test' or 'caserun'. + + Args: + 'scan' : object + 'test' : object, + 'caserun' : object, + 'threshold' : int + } + + Use `Issuer.build_issue()` to generate new `Issue` + + Returns: None + """ + + + + + def __init__( + self, + scan : object=None, + test : object=None, + caserun : object=None, + threshold : int=75 + ): + + # main objects + self.scan = scan + self.test = test + self.caserun = caserun + self.object = None + self.type = None + self.threshold = test.threshold if test else threshold + + # top level vars + self.title = None + self.details = None + self.data = None + self.labels = None + self.account = None + self.trigger = { 'type': None, 'id': None } + self.affected = { 'type': None, 'id': None, 'str': None} + self.max_len = 200 + self.max_tokens = 5000 + self.gpt_model = "gpt-4o-mini" + + # init GPT client + self.gpt_client = OpenAI( + api_key=settings.GPT_API_KEY, + ) + + + + + def convert_key(self, key: str=None) -> str: + """ + Converts the passed camel case or + snake case str into a spaced str with + each word capitalized + + Args: + key: str + } + + Returns: str + """ + + # remove "_delta" + key = key.replace('_delta', '') + + # convert from camelCase + key = re.sub(r"([a-z])([A-Z])", r"\1 \2", key) + + # convert snake_case + key = key.replace('_', ' ') + + # capitalize + key = key.title() + + return key + + + + + def clean_recommendation(self, recommendation: str=None) -> str: + """ + Replaces URLs with correct URLs + + Args: + 'recommendation': str + } + + Returns: str + """ + # clean client URI + client_uri = settings.CLIENT_URL_ROOT.split('://')[1] + + # repalce URI + recommendation = recommendation.replace( + 'localhost', + client_uri + ) + + # return clean recommendation + return recommendation + + + + + def build_issue(self): + """ + Creates a new `Issue` based on the info + from the passed "self.test" or "self.caserun" + + Expects: None + + Returns: `Issue` + """ + + # deciding on type + self.obj = self.scan or self.test or self.caserun + self.type = 'scan' if self.scan else 'test' if self.test else 'caserun' + + # define account + self.account = self.obj.site.account + + # update triggers & affected + self.trigger = { + 'type' : self.type, + 'id' : str(self.obj.id) + } + self.affected = { + 'type' : 'site' if self.caserun else 'page', + 'id' : str(self.obj.site.id) if self.caserun else str(self.obj.page.id), + 'str' : self.obj.site.site_url if self.caserun else self.obj.page.page_url + } + + # building details, title, & recomemdations + # for Scan performance & log data + if self.scan: + self._handle_scan() + + # building details, title, & + # recomemdations for test failure + if self.test: + self._handle_test() + + # building details, title, & + # recomemdations for caserun failure + if self.caserun: + self._handle_caserun() + + # build recommendation + response = self.build_recommendation() + + recommendation = str( + f'\n\n### Recommendations:\n' + + f'{response}' + ) + + # clean recommendation + recommendation = self.clean_recommendation(recommendation) + + # build details from components + self.details = self.details + recommendation + + # creating new Issue + issue = Issue.objects.create( + account = self.account, + title = self.title, + details = self.details, + labels = self.labels, + trigger = self.trigger, + affected = self.affected + ) + + # meter account if necessary + if self.account.type == 'cloud' and self.account.cust_id: + meter_account(str(self.account.id), 2) + + # new Issue + return issue + + + + + def _handle_scan(self) -> None: + """ + Handles data collection for a scan + + Expects: None + + Returns: None + """ + + # defaults for data categorization + cats = [] + comps = [] + lh = [] + yl = [] + logs = [] + + # get raw audits + lh_audits = requests.get(self.scan.lighthouse.get('audits')).json() if self.scan.lighthouse.get('audits') else '' + yl_audits = requests.get(self.scan.yellowlab.get('audits')).json() if self.scan.yellowlab.get('audits') else '' + + # include logs + if self.scan.logs: + if len(self.scan.logs) > 0: + cats.append({ + 'key' : 'logs', + 'name' : 'Console Issues', + 'value' : str(len(self.scan.logs)) + }) + comps.append('Console') + logs = self.scan.logs + + # include lighthouse + if self.scan.lighthouse.get('audits'): + for key in self.scan.lighthouse.get('scores'): + if key != 'average' and key != 'crux': + if int(self.scan.lighthouse.get('scores')[key]) < self.threshold: + # include LH categories + cats.append({ + 'key' : key, + 'name' : f"{self.convert_key(key)} (lighthouse)", + 'value' : f"{self.scan.lighthouse.get('scores')[key]}%" + }) + # update str for title + if not any('Performance' in i for i in comps): + comps.append('& Performance' if 'Console' in comps else 'Performance') + # save audits + lh.append({ + key: lh_audits.get(key) + }) + + # include yellowlab + if self.scan.yellowlab.get('audits'): + for key in self.scan.yellowlab.get('scores'): + if key != 'globalScore': + if int(self.scan.yellowlab.get('scores')[key]) < self.threshold: + # include YL categories + cats.append({ + 'key' : key, + 'name' : f"{self.convert_key(key)} (yellowlab)", + 'value' : f"{self.scan.yellowlab.get('scores')[key]}%" + }) + # update str for title + if not any('Performance' in i for i in comps): + comps.append('& Performance' if 'Console' in comps else 'Performance') + # save audits + yl.append({ + key: yl_audits.get(key) + }) + + # build title + self.title = f'Scan found {' '.join(comps)} Issues' + + # build intro + intro = str( + f'## This [Scan]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) ' + + f'contains {' '.join(comps)} issues.\n' + + f'\n\n> Affected Page ' + + f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]}) \n\n' + ) + + # build components str + comp_str = str('| Component | Value |\n|:-----|-----:|') + for item in cats: + comp_str += f'\n| {item.get('name')} | {item.get('value')} |' + + # build main_issue + main_issue = str( + f'### Failing Components:\n' + + f'{comp_str}' + ) + + # combine into details + self.details = str(intro + main_issue) + + # build & format data for AI + self.data = f'\n\n------------\n\n' + if len(logs) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nBrowser Console Errors and Warnings:' + + f'\n{logs}' + ) + if len(lh) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nAudit data from Google Lighthouse:' + + f'\n{lh}' + ) + if len(yl) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nAudit data from YellowLab Tools:' + + f'\n{yl}' + ) + self.data += f'\n\n------------\n\n' + + + + + def _handle_test(self) -> None: + """ + Handles data collection for a test + + Expects: None + + Returns: None + """ + + # defaults + lh = [] + yl = [] + logs = [] + vrt = {} + lh_audits = '' + yl_audits = '' + + # get post_logs_delta + logs = self.test.logs_delta.get('post_logs_delta') if self.test.logs_delta else [] + + # get raw audits + if self.test.lighthouse_delta: + lh_audits = requests.get(self.test.lighthouse_delta.get('audits')).json() if self.test.lighthouse_delta.get('audits') else '' + if self.test.yellowlab_delta: + yl_audits = requests.get(self.test.yellowlab_delta.get('audits')).json() if self.test.yellowlab_delta.get('audits') else '' + + # record only audits from LH components + # that had negative scores + if (self.test.lighthouse_delta or {}).get('audits'): + if self.test.component_scores.get('lighthouse') < self.threshold: + for key in self.test.lighthouse_delta.get('scores'): + if 'average' not in key and 'crux' not in key: + if self.test.lighthouse_delta.get('scores')[key] < 0: + key = key.replace('_delta', '') + lh.append({ + key: lh_audits.get(key) + }) + + # record only audits from YL components + # that had negative scores + if (self.test.yellowlab_delta or {}).get('audits'): + if self.test.component_scores.get('yellowlab') < self.threshold: + for key in self.test.yellowlab_delta.get('scores'): + if 'average' not in key: + if self.test.yellowlab_delta.get('scores')[key] < 0: + key = key.replace('_delta', '') + yl.append({ + key: yl_audits.get(key) + }) + + # record any information about VRT + if 'vrt' in self.test.type: + vrt['similarity_score'] = self.test.component_scores.get('vrt') + vrt['summary'] = self.test.images_delta.get('summary') + vrt['broken'] = self.test.images_delta.get('broken') + + # grabbing component scores + # which were less than the test.threshold + ordered_scores = [] + for key in self.test.component_scores: + if self.test.component_scores[key] is not None: + if self.test.component_scores[key] < self.test.threshold: + ordered_scores.append({key: self.test.component_scores[key]}) + + # build components str + comp_str = str('| Component | Score |\n|:-----|-----:|') + for score in ordered_scores: + for key in score: + comp_str += f'\n| {key} | {round(score[key], 2)} |' + + # adjusting component names in table + comp_str = comp_str.replace( + 'vrt', + 'visual regression (vrt)' + ).replace( + 'html', + 'html regression (html)' + ) + + # build title + self.title = f'Test Failed at {round(self.test.score, 2)}%' + + # build intro + intro = str( + f'## This [Test]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]}) failed ' + + f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n' + + f'\n\n> Affected Page ' + + f'[{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]})\n\n' + ) + + # build main_issue + main_issue = str( + f'### Failing Components:\n' + + f'{comp_str}' + ) + + # combine into details + self.details = str(intro + main_issue) + + # build & format data for AI + self.data = f'\n\n------------\n\n' + if len(logs) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nBrowser Console Errors and Warnings:' + + f'\n{logs}' + ) + if len(lh) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nAudit data from Google Lighthouse:' + + f'\n{lh}' + ) + if len(yl) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nAudit data from YellowLab Tools:' + + f'\n{yl}' + ) + if len(vrt) > 0: + self.max_len += 50 + self.data += str( + f'\n\n\nVisual Regression Data:' + + f'\n{vrt}' + ) + self.data += f'\n\n------------\n\n' + + + + + def _handle_caserun(self) -> None: + """ + Handles data collection for a caserun + + Expects: None + + Returns: None + """ + # get first step that failed in caserun + failed_step = None + step_index = 0 + step_type = 'action' + for step in self.caserun.steps: + step_index += 1 + if step['action']['status'] == 'failed': + failed_step = step + step_type = 'action' + break + if step['assertion']['status'] == 'failed': + failed_step = step + step_type = 'assertion' + break + + # build title + self.title = f'Case Run "{self.caserun.title}" Failed' + + # build intro + intro = str( + f'## Case Run [{self.caserun.title}]({settings.CLIENT_URL_ROOT}/{self.trigger["type"]}/{self.trigger["id"]})' + + f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' + + f' > Affected Site: [{self.affected["str"]}]({settings.CLIENT_URL_ROOT}/{self.affected["type"]}/{self.affected["id"]})\n\n\n' + ) + + # build main_issue + main_issue = str( + f'### Main Issue or Exception:\n' + + f' ```shell\n{failed_step[step_type]["exception"]}\n``` \n\n' + + f' [View Image]({failed_step[step_type]["image"]})\n\n' + ) + + # combine into details + self.details = str(intro + main_issue) + + # no extra data yet + self.data = None + + + + + def build_recommendation(self) -> str: + """ + Using OpenAI's Chat GPT, composes a personalized + `recommendation` for the primary `Issue` being created. + + Expects: None + + Returns: str + """ + + # initializing + recommendation = '' + + # truncate self.data + if self.data: + encoding = tiktoken.encoding_for_model(self.gpt_model) + tokens = encoding.encode(self.data) + if len(tokens) > self.max_tokens: + tokens = tokens[:self.max_tokens] + self.data = encoding.decode(tokens) + + # building recommendation + # for self.scan + if self.scan: + + # send the initial request + recommendation = self.gpt_client.chat.completions.create( + model=self.gpt_model, # old model -> gpt-3.5-turbo + messages=[ + { + "role": "user", + "content": f"Create a recommendation for developers \ + baseded on this generated issue: '\n\n{self.details}\n\n'. \ + Below is output data from the Scan to help you identify potential recommendations: {self.data} \ + If possible, include reference to any files, scripts, images, etc. which should be addressed based on the provided data. \ + Prioritize recommendations based on highest impact to the performance and security of the web page. \ + Format each recommendation with markdown. \ + Begin each recommendation with '- [ ]' to format as a task. \ + Omit the title or header in your response. \ + Omit any summary after the recommendations. \ + Remove any disclaimer or note section. \ + Remove any reference to 'Test Cases'. \ + Remove and reference to 'visual comparison tools'. \ + Max Length of Response: {self.max_len} words. \ + Tone: Instructive" + }, + ] + ).choices[0].message.content + + + # building recommendation + # for self.test + if self.test: + + # send the initial request + recommendation = self.gpt_client.chat.completions.create( + model=self.gpt_model, + messages=[ + { + "role": "user", + "content": f"Create a recommendation for developers \ + baseded on this generated issue: '\n\n{self.details}\n\n'. \ + The components are portions of a regression test of a website. \ + Below is output data from the Test to help you identify potential \ + recommendations for performance, browser logs, and visual regressions: {self.data} \ + If possible, include reference to any files, scripts, etc. \ + which should be addressed based on the provided data. \ + Format each recommendation with markdown. \ + Begin each recommendation with '- [ ]' to format as a task. \ + Omit the title or header in your response. \ + Omit any summary after the recommendations. \ + Remove any disclaimer or note section. \ + Remove any reference to 'Test Cases'. \ + Remove and reference to 'visual comparison tools'. \ + Max Length of Response: {self.max_len} words. \ + Tone: Instructive" + }, + ] + ).choices[0].message.content + + + # building recommendation + # for self.caserun + if self.caserun: + + # send the initial request + recommendation = self.gpt_client.chat.completions.create( + model=self.gpt_model, + messages=[ + { + "role": "user", + "content": f"Create a recommendation for developers \ + baseded on this generated issue: '\n\n{self.details}\n\n'. \ + Format each recommendation with markdown. \ + Begin each recommendation with '- [ ]' to format as a task. \ + Omit any summary after the recommendations. \ + Omit the title or header in your response. \ + Omit any links in your response. \ + Remove any disclaimer or notes section. \ + Remove any reference to selenium documentation. \ + Remove any reference of 'alternative selector strategies'. \ + Max Length of Response: {self.max_len} words. \ + Tone: Instructive" + }, + ] + ).choices[0].message.content + + # return recommendation + return recommendation + + + + + diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py index ad917045..bae646c3 100644 --- a/app/api/utils/lighthouse.py +++ b/app/api/utils/lighthouse.py @@ -1,155 +1,368 @@ -import subprocess, json +from pathlib import Path from ..models import Site, Scan +from .devices import get_device +from cursion import settings +import subprocess, json, uuid, boto3, os, requests, ast + + + class Lighthouse(): - """Initializes Google's Lighthouse CLI and runs an audit of the site""" + """ + Initializes Google's Lighthouse CLI and runs an audit of the site + + Use self.get_data() to init a run + """ + + + def __init__(self, scan=None): + self.scan = scan + self.site = self.scan.site + self.page = self.scan.page + self.configs = scan.configs + self.sizes = scan.configs['window_size'].split(',') + self.cpu_slowdown = 1 + self.scale_factor = 2 + self.audits_url = '' + self.device = get_device( + scan.configs['browser'], + scan.configs['device'] + ) + self.is_mobile = str(self.device['type'] == 'mobile' or self.device['type'] == 'tablet').lower() + + # device specific network speeds + self.speed = { + 'mobile': { + 'download': 4000, + 'upload': 1000, + 'rttMs': 40 + }, + 'tablet': { + 'download': 4000, + 'upload': 1000, + 'rttMs': 40 + }, + 'desktop': { + 'download': 12000, + 'upload': 5500, + 'rttMs': 10 + } + } + + # initial scores object + self.scores = { + "seo": None, + "accessibility": None, + "performance": None, + "best_practices": None, + # "pwa": None, + "crux": None, + "average": None + } + + # initial audits object + self.audits = { + "seo": [], + "accessibility": [], + "performance": [], + "best_practices": [], + # "pwa": [], + "crux": [] + } + + + + + def create_configs(self): + + # custom Lighthouse config + config_js = f""" + module.exports = {{ + extends: 'lighthouse:default', + plugins: ['lighthouse-plugin-crux'], + settings: {{ + cruxToken: "{settings.GOOGLE_CRUX_KEY}", + skipAudits: [ + "full-page-screenshot" + ], + screenEmulation: {{ + mobile: {self.is_mobile}, + width: {self.sizes[0]}, + height: {self.sizes[1]}, + deviceScaleFactor: {self.scale_factor}, + disabled: false + }}, + throttling: {{ + cpuSlowdownMultiplier: {self.cpu_slowdown} + }}, + emulatedUserAgent: {json.dumps(self.device['user_agent'])} + }} + }}; + """ + + # define output path + config_path = Path("api/utils/configs/custom-config.js") + config_path.write_text(config_js) + + # return path for use in subprocess + return config_path.as_posix() - def __init__(self, site=None, configs=None): - self.site = site - self.configs = configs - self.sizes = configs['window_size'].split(',') - def init_audit(self): + def lighthouse_cli(self): + """ + Serves as the CLI method for collecting LH metrics. + Creates a sub process running lighthouse CLI + + Returns: raw LH data (Dict) + """ + + # warm up the page by curl'ing site + try: + subprocess.run( + ['curl', '-sS', '--max-time', '5', self.page.page_url], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True + ) + except subprocess.CalledProcessError: + pass + + # initiating subprocess for LH CLI proc = subprocess.Popen([ 'lighthouse', - '--config-path=api/utils/custom-config.js', + f'--config-path=api/utils/configs/default-config.js', '--quiet', - self.site.site_url, + self.page.page_url, '--plugins=lighthouse-plugin-crux', - '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', + '--extra-headers=api/utils/configs/extra-headers.json', + f'--chrome-flags=--no-sandbox --headless --disable-dev-shm-usage', + f'--form-factor={self.device["type"]}', f'--screenEmulation.width={self.sizes[0]}', f'--screenEmulation.height={self.sizes[1]}', - f'--screenEmulation.{self.configs["device"]}', + f'--screenEmulation.mobile={self.is_mobile}', + f'--emulatedUserAgent={self.device["user_agent"]}', + f'--throttling.cpuSlowdownMultiplier={self.cpu_slowdown}', + f'--throttling.downloadThroughputKbps={self.speed[self.device["type"]]["download"]}', + f'--throttling.uploadThroughputKbps={self.speed[self.device["type"]]["upload"]}', + f'--throttling.rttMs={self.speed[self.device["type"]]["rttMs"]}', + f'--throttling-method=devtools', '--output', - 'json', - ], + 'json', + ], stdout=subprocess.PIPE, user='app', ) - stdout_value = proc.communicate()[0] - return stdout_value - - def get_data(self): + # retrieving data from process + stdout_value = proc.communicate()[0] + + # decode bytes into string + stdout_string = stdout_value.decode('iso-8859-1') + # clean string of any errors try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') - - # clean string of any errors delm = '{\n "lighthouseVersion"' stdout_string = delm + stdout_string.split(delm)[1] + except Exception as e: + print(e) + + # encode back to bytes + stdout_value = stdout_string.encode('iso-8859-1') + + # converting stdout str into Dict + try: + stdout_json = json.loads(stdout_value) + except json.JSONDecodeError: + # fallback for Python dict literal debugging payloads + stdout_json = ast.literal_eval(stdout_string) + return stdout_json - # encode back to bytes - stdout_value = stdout_string.encode('iso-8859-1') - - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'lighthouse ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best-practices": [], - "lighthouse-plugin-crux": [], - "pwa": [] - } - - # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["categories"].get(cat).get("auditRefs") - if cat_audits is not None: - for a in cat_audits: - if int(a["weight"]) > 0: - audit = stdout_json["audits"][a["id"]] - audits[cat].append(audit) - # changing audits names - audits['best_practices'] = audits.pop('best-practices') - audits['crux'] = audits.pop('lighthouse-plugin-crux') - - # get scores from each category - seo_score = round(stdout_json["categories"]["seo"]["score"] * 100) - accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100) - performance_score = round(stdout_json["categories"]["performance"]["score"] * 100) - best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100) - pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100) - - # attempting crux - try: - crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100) - except: - crux_score = 0 - - if crux_score == 0 : - crux_score = None - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score - )/ 5) - else: - average_score = round(( - seo_score + accessibility_score + performance_score - + best_practices_score + pwa_score + crux_score - )/ 6) - - scores = { - "seo": seo_score, - "accessibility": accessibility_score, - "performance": performance_score, - "best_practices": best_practices_score, - "pwa": pwa_score, - "crux": crux_score, - "average": average_score - } - - - data = { - "scores": scores, - "audits": audits, - "failed": False - } - - else: - raise RuntimeError + + + def lighthouse_api(self) -> dict: + """ + Serves as the API method for collecting LH metrics. + Sends API requests to + + Returns: raw LH data (Dict) + """ + + # defaults + headers = { + "content-type": "application/json", + } + params = { + "url": self.page.page_url, + "strategy": self.device['type'], + "key": settings.GOOGLE_CRUX_KEY + } + + # cats + cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=SEO' + + # setting up initial request + res = requests.get( + url=f'{settings.LIGHTHOUSE_ROOT}?{cats}', + params=params, + headers=headers + ) + + # print error if not 200 + if not str(res.status_code).startswith('2'): + print(res.status_code, res.text) + + # try to get just LH response + res_json = res.json() + res = res_json.get('lighthouseResult') + + # return response + return res + + + + + def process_data(self, stdout_json: dict) -> dict: + """ + Accepts JSON data from either CLI or API method + and parses into usable Cursion data. + + Expects the following: + stdout_json: or json from output - except Exception as e: - print(e) + Returns: formatted LH data + """ - scores = { - "seo": None, - "accessibility": None, - "performance": None, - "best_practices": None, - "pwa": None, - "crux": None, - "average": None - } + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # allow_list of 0 weighted audits + allow_list = [ + 'server-response-time', 'cache-insight', + 'interactive', + ] - audits = { - "seo": [], - "accessibility": [], - "performance": [], - "best_practices": [], - "pwa": [], - "crux": [] + try: + # Map internal keys (used by the client) to Lighthouse category keys. + category_key_map = { + "best_practices": "best-practices", + "crux": "lighthouse-plugin-crux", } + # iterating through categories to get relevant lh_audits + # and store them in their respective `audits = {}` obj + for cat in self.audits: + lh_cat = category_key_map.get(cat, cat) + # skipping non-existent cat + if stdout_json["categories"].get(lh_cat) is None: + continue + cat_audits = stdout_json["categories"].get(lh_cat).get("auditRefs") + if cat_audits is not None: + for a in cat_audits: + if int(a["weight"]) > 0 or a["id"] in allow_list: + audit = stdout_json["audits"][a["id"]] + self.audits[cat].append(audit) + + # get scores from each category + score_queue = [] + for cat in self.scores: + lh_cat = category_key_map.get(cat, cat) + # skipping non-existent cat + if stdout_json["categories"].get(lh_cat) is None: + continue + score_value = stdout_json["categories"][lh_cat]["score"] + if score_value is None: + continue + # record score + self.scores[cat] = round(score_value * 100) + # add to queue + score_queue.append(self.scores[cat]) + + # dynamically calculating average + if score_queue: + average_score = round(sum(score_queue)/len(score_queue)) + self.scores['average'] = average_score + + + # save audits data as json file + file_id = uuid.uuid4() + audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json') + with open(audit_file, 'w') as fp: + json.dump(self.audits, fp) + + # upload to s3 and return url + remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json' + root_path = settings.AWS_S3_URL_PATH + self.audits_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(audit_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + # remove local copy + os.remove(audit_file) + data = { - "scores": scores, - "audits": audits, - "failed": True + "scores": self.scores, + "audits": self.audits_url, + "failed": False } - + + # returning data + return data + + except Exception as e: + print(f'FAILED to pasrse: {e.__class__.__name__}: {e}\n{stdout_json}') + raise TypeError + + + + + def get_data(self): + + scan_complete = False + failed = True + attempts = 0 + + # trying lighthouse scan untill success or 2 attempts + while not scan_complete and attempts < 2: + + try: + # CLI on first attempt if not API Priority + if attempts < 1 and not self.configs.get('api_priority'): + raw_data = self.lighthouse_cli() + self.process_data(stdout_json=raw_data) + + # API after first attempt or if API Priority + if attempts >= 1 or self.configs.get('api_priority'): + raw_data = self.lighthouse_api() + self.process_data(stdout_json=raw_data) + + scan_complete = True + failed = False + + except Exception as e: + print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}') + scan_complete = False + failed = True + attempts += 1 + + data = { + "scores": self.scores, + "audits": self.audits_url if self.audits_url != '' else None, + "failed": failed + } + + # returning final data return data diff --git a/app/api/utils/manager.py b/app/api/utils/manager.py new file mode 100644 index 00000000..4e4f6d99 --- /dev/null +++ b/app/api/utils/manager.py @@ -0,0 +1,76 @@ +from ..models import * +from cursion import settings + + + + + + +def record_task( + resource_type: str=None, + resource_id: str=None, + task_id: str=None, + task_method: str=None, + **kwargs, + ) -> bool: + """ + Records task information in the `resource.system` + attribute. + + Args: + 'resource_type' : str (scan, test, caserun) + 'resource_id' : str + 'task_id' : str + 'task_method' : str + 'kwargs' : dict + + Returns: max_attempts_reached + """ + + # set default + max_attempts_reached = False + + # get resource + if resource_type == 'scan': + resource = Scan.objects.get(id=resource_id) + if resource_type == 'test': + resource = Test.objects.get(id=resource_id) + if resource_type == 'caserun': + resource = CaseRun.objects.get(id=resource_id) + + # get current resoruce.system.tasks data + tasks = (resource.system or {}).get('tasks', []) + + # get component based on task_name + component = task_method.replace('run_', '').replace('_bg', '').replace('_and_logs', '') + + # check if task exists + i = 0 + exists = False + for task in tasks: + if task['component'] == component: + # update existing task + max_attempts_reached = True if (tasks[i]['attempts'] >= settings.MAX_ATTEMPTS) else False + tasks[i]['task_id'] = str(task_id) + tasks[i]['attempts'] += 1 if not max_attempts_reached else tasks[i]['attempts'] + tasks[i]['kwargs'] = kwargs.get('kwargs') + exists = True + i += 1 + + # append new task data + if not exists: + tasks.append({ + 'attempts' : int(1), + 'task_id' : str(task_id), + 'task_method' : str(task_method), + 'component' : str(component), + 'kwargs' : kwargs.get('kwargs'), + }) + + # update resource with new system data + resource.system = resource.system or {} + resource.system['tasks'] = tasks + resource.save() + + # return + return max_attempts_reached \ No newline at end of file diff --git a/app/api/utils/meter.py b/app/api/utils/meter.py new file mode 100644 index 00000000..416991d3 --- /dev/null +++ b/app/api/utils/meter.py @@ -0,0 +1,38 @@ +from ..models import Account +from cursion import settings +import stripe + + + + + + +def meter_account(account_id: str=None, count: int=1) -> None: + """ + Sends a `MeterEvent` request to Stripe to + track account usage + + Expects: + 'account_id' `` (REQUIRED) + 'count' `` (OPTIONAL) + + Returns: None + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get account + account = Account.objects.get(id=account_id) + + # send stripe request + stripe.billing.MeterEvent.create( + event_name = 'tasks', + payload = { + 'stripe_customer_id': account.cust_id, + 'value': count + }, + ) + + # return + return None \ No newline at end of file diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py index 77dffb4a..57521afa 100644 --- a/app/api/utils/reporter.py +++ b/app/api/utils/reporter.py @@ -1,466 +1,825 @@ -from ..models import * -import time, os, sys, json, boto3 -import PIL.Image as Img -from scanerr import settings -from datetime import datetime, timedelta +from datetime import timedelta +import os +import textwrap + +import boto3 +from django.utils import timezone +from reportlab.graphics import renderPDF +from reportlab.graphics.charts.barcharts import VerticalBarChart +from reportlab.graphics.charts.piecharts import Pie +from reportlab.graphics.shapes import Drawing, String +from reportlab.lib.colors import HexColor from reportlab.lib.pagesizes import letter from reportlab.lib.units import inch -from reportlab.lib.colors import HexColor from reportlab.pdfgen import canvas +from ..models import CaseRun, Issue, Page, Scan, Site, Test +from cursion import settings -class Reporter(): - - ''' - Used for generating web vitals reports for the passed `Site` obj +class Reporter: + """ + Generates a site-level PDF report for the associated `Report` object. - Expects -> { - "report": , - } + Returns: + 'report' : object, + 'success': bool, + 'message': str + """ - returns --> - - ''' + VALID_LOOKBACK_DAYS = {1, 7, 30, 90} + VALID_TYPES = {"issues", "tests", "caseruns", "performance"} + FONT_REGULAR = "Helvetica" + FONT_BOLD = "Helvetica-Bold" - def __init__(self, report, scan=None): + def __init__(self, report: object, scan: object = None): self.report = report + self.scan = scan self.site = self.report.site - if scan is None: - self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id']) - else: - self.scan = scan - #building paths & canvas template - if os.path.exists(os.path.join(settings.BASE_DIR, f'temp/')): - self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') - else: - os.makedirs(f'{settings.BASE_DIR}/temp') - self.local_path = os.path.join(settings.BASE_DIR, f'temp/{self.report.id}.pdf') - + if self.site is None and self.report.page is not None: + self.site = self.report.page.site + + info = self.report.info if isinstance(self.report.info, dict) else {} + self.text_color = info.get("text_color", "#24262d") + self.highlight_color = info.get("highlight_color", "#ffffff") + self.background_color = info.get("background_color", "#e1effd") + + reports_dir = os.path.join(settings.BASE_DIR, "reports") + if not os.path.exists(reports_dir): + os.makedirs(reports_dir) + self.local_path = os.path.join(reports_dir, f"{self.report.id}.pdf") + self.page_index = 0 - self.text_color = self.report.info['text_color'] - self.highlight_color = self.report.info['highlight_color'] - self.background_color = self.report.info['background_color'] self.c = canvas.Canvas(self.local_path, letter) - self.y = 9 - - def setup_page(self): - # sets the defaults for a new page + self.s3 = boto3.client( + "s3", + aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL), + ) + + + def _with_alpha(self, color_hex: str, alpha_hex: str) -> HexColor: + color_hex = str(color_hex or "#000000").strip() + if not color_hex.startswith("#"): + color_hex = f"#{color_hex}" + if len(color_hex) != 7: + color_hex = "#000000" + return HexColor(f"{color_hex}{alpha_hex}", hasAlpha=True) + + + def _fit_text(self, text: str, font_name: str, font_size: float, max_width_px: float) -> str: + value = str(text or "") + if self.c.stringWidth(value, font_name, font_size) <= max_width_px: + return value + + suffix = "..." + low = 0 + high = len(value) + best = "" + while low <= high: + mid = (low + high) // 2 + candidate = f"{value[:mid]}{suffix}" + if self.c.stringWidth(candidate, font_name, font_size) <= max_width_px: + best = candidate + low = mid + 1 + else: + high = mid - 1 + return best or suffix + + + def _wrap_text_to_width(self, text: str, font_name: str, font_size: float, max_width_px: float) -> list[str]: + value = str(text or "").replace("\n", " ").strip() + if not value: + return [""] + + words = value.split() + lines: list[str] = [] + current = "" + + for word in words: + candidate = f"{current} {word}".strip() + if current and self.c.stringWidth(candidate, font_name, font_size) > max_width_px: + lines.append(current) + current = word + if self.c.stringWidth(current, font_name, font_size) > max_width_px: + chunk = "" + for char in current: + next_chunk = f"{chunk}{char}" + if chunk and self.c.stringWidth(next_chunk, font_name, font_size) > max_width_px: + lines.append(chunk) + chunk = char + else: + chunk = next_chunk + current = chunk + else: + current = candidate + + if current: + lines.append(current) + + return lines or [""] + + + def setup_page(self) -> None: self.c.setFillColor(HexColor(self.background_color)) - self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1) + self.c.rect(0, 0, 8.5 * inch, 11 * inch, stroke=0, fill=1) - def end_page(self): - # adds page number and ends page - self.c.setFont('Helvetica-Bold', 15) + def end_page(self) -> None: + self.c.setFont(self.FONT_BOLD, 10) self.c.setFillColor(HexColor(self.text_color)) self.page_index += 1 - self.c.drawString(7.7*inch, .3*inch, str(self.page_index)) + self.c.drawString(7.7 * inch, 0.3 * inch, str(self.page_index)) self.c.showPage() - def draw_page_title(self, title): - # adds a title to the given page - self.c.setFont('Helvetica-Bold', 32) + def draw_page_title(self, title: str, subtitle: str | None = None) -> None: + self.c.setFont(self.FONT_BOLD, 27) self.c.setFillColor(HexColor(self.text_color)) - self.c.drawCentredString(4.25*inch, 10*inch, title) + self.c.drawString(0.5 * inch, 9.9 * inch, title) + if subtitle: + self.c.setFont(self.FONT_REGULAR, 11) + self.c.drawString(0.5 * inch, 9.56 * inch, subtitle) - def publish_report(self): - self.c.save() - remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf' - s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) + def draw_wrapped_line(self, text: str, length: int, x_pos: float, y_pos: float, y_offset: float) -> float: + wraps = textwrap.wrap(str(text), length, break_long_words=True) or [""] + for line in wraps: + self.c.drawString(x_pos * inch, y_pos * inch, line) + y_pos -= y_offset + return y_pos - # uploading package to remote s3 - with open(self.local_path, 'rb') as data: - s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), - remote_path, ExtraArgs={ - 'ACL': 'public-read', 'ContentType': 'application/pdf'} - ) - report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0' + def _draw_stat_card(self, x: float, y: float, w: float, h: float, title: str, value: str, subtitle: str = "", value_font_size: float = 17) -> None: + self.c.setFillColor(self._with_alpha(self.highlight_color, "CF")) + self.c.roundRect(x * inch, y * inch, w * inch, h * inch, 0.1 * inch, stroke=0, fill=1) - self.report.path = report_url - self.report.save() - os.remove(self.local_path) + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont(self.FONT_REGULAR, 9) + self.c.drawString((x + 0.14) * inch, (y + h - 0.22) * inch, title) - + fitted_value = self._fit_text(value, self.FONT_BOLD, value_font_size, (w - 0.28) * inch) + self.c.setFont(self.FONT_BOLD, value_font_size) + self.c.drawString((x + 0.14) * inch, (y + 0.34) * inch, fitted_value) + + if subtitle: + self.c.setFont(self.FONT_REGULAR, 8) + self.c.drawString((x + 0.14) * inch, (y + 0.13) * inch, self._fit_text(subtitle, self.FONT_REGULAR, 8, (w - 0.28) * inch)) - def cover_page(self): - # background and title - self.setup_page() - - # creating dark triangle - p = self.c.beginPath() - p.moveTo(0*inch, 11*inch) - p.lineTo(7*inch, 11*inch) - p.lineTo(2.5*inch, 4.5*inch) - p.lineTo(0*inch, 7*inch) - self.c.setFillColor(HexColor('#00000026', hasAlpha=True)) - self.c.setStrokeColor(HexColor('#00000026', hasAlpha=True)) - self.c.drawPath(p, fill=1) - - # crating light triangle - p = self.c.beginPath() - p.moveTo(0*inch, 0*inch) - p.lineTo(0*inch, 7*inch) - p.lineTo(7*inch, 0*inch) - self.c.setFillColor(HexColor('#0000000D', hasAlpha=True)) - self.c.setStrokeColor(HexColor('#0000000D', hasAlpha=True)) - self.c.drawPath(p, fill=1) - - # date - date = f'{self.scan.time_created.month}/{self.scan.time_created.day}/{self.scan.time_created.year}' - self.c.setFont('Helvetica-Bold', 24) - self.c.setFillColor(HexColor(self.text_color)) - self.c.drawString(.5*inch, 7.5*inch, date) - # title - self.c.setFont('Helvetica-Bold', 45) + def _draw_pie_chart(self, x: float, y: float, w: float, h: float, title: str, data_pairs: list[tuple[str, float]]) -> None: + values = [float(max(0, p[1])) for p in data_pairs] + labels = [str(p[0]) for p in data_pairs] + if not values or sum(values) <= 0: + self.c.setFont(self.FONT_REGULAR, 10) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(x * inch, y * inch, f"{title}: no data") + return + + drawing = Drawing(w * inch, h * inch) + pie = Pie() + pie_bottom = 0.08 * inch + pie_top = (h * inch) - 0.55 * inch + max_pie_width = (w - 0.2) * inch + max_pie_height = max(0.6 * inch, pie_top - pie_bottom) + pie_size = min(max_pie_width, max_pie_height) + pie.x = ((w * inch) - pie_size) / 2 + pie.y = pie_bottom + ((max_pie_height - pie_size) / 2) + pie.width = pie_size + pie.height = pie_size + pie.data = values + pie.labels = labels + pie.slices.strokeWidth = 0.5 + pie.slices.fontName = self.FONT_REGULAR + pie.slices.fontSize = 8 + + palette = [ + HexColor("#38B43F"), + HexColor("#DB524B"), + HexColor("#E3A635"), + HexColor("#4B79DB"), + HexColor("#9A5FDB"), + HexColor("#4DB2A7"), + ] + for i in range(len(values)): + pie.slices[i].fillColor = palette[i % len(palette)] + + drawing.add(pie) + drawing.add(String(4, h * inch - 12, title, fontName=self.FONT_BOLD, fontSize=10, fillColor=HexColor(self.text_color))) + renderPDF.draw(drawing, self.c, x * inch, y * inch) + + + def _draw_bar_chart(self, x: float, y: float, w: float, h: float, title: str, labels: list[str], values: list[float]) -> None: + if not values: + self.c.setFont(self.FONT_REGULAR, 10) + self.c.setFillColor(HexColor(self.text_color)) + self.c.drawString(x * inch, y * inch, f"{title}: no data") + return + + data_max = max(float(v) for v in values) + if data_max <= 0: + axis_max = 1.0 + elif data_max <= 5: + axis_max = data_max + 1 + else: + axis_max = data_max * 1.1 + axis_step = max(1.0, round(axis_max / 5)) + + chart_w_px = w * inch + chart_h_px = h * inch + title_y = chart_h_px - 12 + + indexed_labels = [str(i + 1) for i in range(len(labels))] + legend_font_size = 7 + legend_line_h = 0.1 * inch + legend_item_gap = 0.03 * inch + legend_top_margin = 0.18 * inch + legend_bottom_gap = 0.1 * inch + legend_cols = 1 if len(labels) <= 3 else 2 + legend_gap = 0.16 * inch if legend_cols > 1 else 0 + legend_width = chart_w_px - 0.16 * inch + legend_col_width = max(1.0 * inch, (legend_width - legend_gap) / legend_cols) + + col_split = max(1, (len(labels) + legend_cols - 1) // legend_cols) + legend_entries: list[list[tuple[int, list[str]]]] = [[] for _ in range(legend_cols)] + col_heights = [0.0 for _ in range(legend_cols)] + + for i, label in enumerate(labels): + col = min(i // col_split, legend_cols - 1) + wrapped = self._wrap_text_to_width(f"{i + 1}: {label}", self.FONT_REGULAR, legend_font_size, legend_col_width) + legend_entries[col].append((i + 1, wrapped)) + col_heights[col] += (len(wrapped) * legend_line_h) + legend_item_gap + + legend_height = max(col_heights) if legend_entries else 0 + chart_top = title_y - legend_top_margin - legend_height - legend_bottom_gap + chart_y = 0.4 * inch + chart_height = max(0.95 * inch, chart_top - chart_y) + + drawing = Drawing(w * inch, h * inch) + chart = VerticalBarChart() + chart.x = 0.45 * inch + chart.y = chart_y + chart.width = (w - 0.7) * inch + chart.height = chart_height + chart.data = [values] + chart.valueAxis.valueMin = 0 + chart.valueAxis.valueMax = axis_max + chart.valueAxis.valueStep = axis_step + chart.valueAxis.labels.fontSize = 7 + chart.valueAxis.labels.fontName = self.FONT_REGULAR + chart.categoryAxis.categoryNames = indexed_labels + chart.categoryAxis.labels.fontSize = 7 + chart.categoryAxis.labels.fontName = self.FONT_REGULAR + chart.categoryAxis.labels.boxAnchor = "n" + chart.barWidth = 0.17 * inch + chart.groupSpacing = 0.14 * inch + chart.barSpacing = 0.05 * inch + chart.bars[0].fillColor = self._with_alpha(self.highlight_color, "E6") + chart.strokeColor = self._with_alpha(self.text_color, "66") + + drawing.add(chart) + drawing.add(String(4, title_y, title, fontName=self.FONT_BOLD, fontSize=10, fillColor=HexColor(self.text_color))) + + legend_y_start = title_y - legend_top_margin + for col, entries in enumerate(legend_entries): + lx = (0.08 * inch) + (col * (legend_col_width + legend_gap)) + ly = legend_y_start + for _, wrapped in entries: + for line in wrapped: + drawing.add(String(lx, ly, line, fontName=self.FONT_REGULAR, fontSize=legend_font_size, fillColor=HexColor(self.text_color))) + ly -= legend_line_h + ly -= legend_item_gap + + renderPDF.draw(drawing, self.c, x * inch, y * inch) + + + def _draw_design_list(self, x: float, y: float, w: float, row_h: float, title: str, items: list[dict], max_rows: int = 6) -> None: + self.c.setFont(self.FONT_BOLD, 10) self.c.setFillColor(HexColor(self.text_color)) - self.c.drawString(.5*inch, 10*inch, 'Web Vitals for') - if len(self.site.site_url) <= 12: - self.c.drawString(.5*inch, 9*inch, self.site.site_url) - elif 12 < len(self.site.site_url): - extra_chars = len(self.site.site_url) - 12 - m = (3/5) - self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m))) + self.c.drawString(x * inch, y * inch, title) + self.c.setStrokeColor(self._with_alpha(self.text_color, "44")) + self.c.line(x * inch, (y - 0.04) * inch, (x + w) * inch, (y - 0.04) * inch) + + y -= 0.15 + cursor_y = y + for idx, item in enumerate(items[:max_rows], start=1): + accent_color = item.get("accent_color", "#4B79DB") + badge = item.get("badge") + badge_w = 0 + if badge: + badge_w = min(2.1, 0.28 + (len(str(badge)) * 0.065)) + + left_padding = 0.11 * inch + right_padding = 0.1 * inch + max_text_width = max(28, (w * inch) - left_padding - right_padding - ((badge_w + 0.18) * inch if badge else 0)) + + headline_font_size = 8.7 + meta_font_size = 8 + headline_line_h = 0.125 * inch + meta_line_h = 0.118 * inch + headline_meta_gap = 0.102 * inch + top_bottom_padding = 0.085 * inch + + headline_lines = self._wrap_text_to_width(item.get("headline", ""), self.FONT_BOLD, headline_font_size, max_text_width) + meta_lines = self._wrap_text_to_width(item.get("meta", ""), self.FONT_REGULAR, meta_font_size, max_text_width) + headline_block_h = max(0, (len(headline_lines) - 1)) * headline_line_h + meta_block_h = max(0, (len(meta_lines) - 1)) * meta_line_h + content_h = headline_block_h + headline_meta_gap + meta_block_h + box_h = max((row_h - 0.04) * inch, content_h + (top_bottom_padding * 2)) + box_h_in = box_h / inch + box_y = cursor_y - box_h_in + box_y_px = box_y * inch + + alpha = "A8" if idx % 2 else "8F" + self.c.setFillColor(self._with_alpha(self.highlight_color, alpha)) + self.c.roundRect(x * inch, box_y_px, w * inch, box_h, 0.05 * inch, stroke=0, fill=1) + + self.c.setFillColor(self._with_alpha(accent_color, "EE")) + self.c.roundRect(x * inch, box_y_px, 0.06 * inch, box_h, 0.03 * inch, stroke=0, fill=1) + + if badge: + badge_x = x + w - badge_w - 0.1 + badge_color = item.get("badge_color", "#4B79DB") + self.c.setFillColor(self._with_alpha(badge_color, "DD")) + badge_h = 0.18 * inch + badge_y = box_y_px + ((box_h - badge_h) / 2) + self.c.roundRect(badge_x * inch, badge_y, badge_w * inch, badge_h, 0.08 * inch, stroke=0, fill=1) + self.c.setFillColor(HexColor("#ffffff")) + self.c.setFont(self.FONT_BOLD, 7.3) + self.c.drawCentredString((badge_x + (badge_w / 2)) * inch, badge_y + (0.055 * inch), str(badge)) + + text_x = (x + 0.11) * inch + line_y = box_y_px + (box_h / 2) + (content_h / 2) + self.c.setFillColor(HexColor(self.text_color)) - self.c.drawString(.5*inch, 9*inch, self.site.site_url) - # cover img - cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png") - self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto') + self.c.setFont(self.FONT_BOLD, headline_font_size) + for i, line in enumerate(headline_lines): + self.c.drawString(text_x, line_y, line) + if i < len(headline_lines) - 1: + line_y -= headline_line_h + + line_y -= headline_meta_gap + self.c.setFont(self.FONT_REGULAR, meta_font_size) + for i, line in enumerate(meta_lines): + self.c.drawString(text_x, line_y, line) + if i < len(meta_lines) - 1: + line_y -= meta_line_h + + cursor_y = box_y - 0.04 + + + def _normalize_types(self): + raw_types = self.report.type + if raw_types is None: + info = self.report.info if isinstance(self.report.info, dict) else {} + raw_types = info.get("types") or info.get("type") + + if isinstance(raw_types, str): + selected = [raw_types] + elif isinstance(raw_types, list): + selected = [item for item in raw_types if isinstance(item, str)] + else: + selected = [] - self.end_page() + selected = list(dict.fromkeys([item.strip().lower() for item in selected if item.strip()])) + if not selected: + return None, "Invalid report config: `type` is required and must be a non-empty array." + invalid = [item for item in selected if item not in self.VALID_TYPES] + if invalid: + return None, f"Invalid report config: unsupported `type` values {invalid}. Supported values: {sorted(self.VALID_TYPES)}." - - def get_score_data(self, score, is_binary=False): - score = float(score) - if is_binary: - score = score*100 + return selected, None - score_types = { - "a": { - "grade": "A", - "color": "#38B43F", - }, - "b": { - "grade": "B", - "color": "#82B436", - }, - "c": { - "grade": "C", - "color": "#ACB43C", - }, - "d": { - "grade": "D", - "color": "#B49836", - }, - "e": { - "grade": "E", - "color": "#B46B34", + + def _get_lookback_days(self): + info = self.report.info if isinstance(self.report.info, dict) else {} + raw_days = info.get("lookback_days") + try: + lookback_days = int(raw_days) + except Exception: + return None, "Invalid report config: `lookback_days` is required (1, 7, 30, or 90)." + + if lookback_days not in self.VALID_LOOKBACK_DAYS: + return None, "Invalid report config: `lookback_days` must be one of 1, 7, 30, 90." + + return lookback_days, None + + + def _resolve_site(self): + if self.site is not None: + return self.site, None + + info = self.report.info if isinstance(self.report.info, dict) else {} + site_id = info.get("site_id") + if not site_id: + return None, "Report generation failed: `site_id` is required for site-level reports." + + try: + site = Site.objects.get(id=site_id) + except Site.DoesNotExist: + return None, f"Report generation failed: site `{site_id}` not found." + + self.site = site + self.report.site = site + return site, None + + + def _build_issues_data(self, window_start, now): + records = [] + _ids = [str(p.id) for p in Page.objects.filter(site=self.site)] + [str(self.site.id)] + issues = ( + Issue.objects.filter(account=self.report.account, status="open", time_created__gte=window_start, affected__id__in=_ids) + .order_by("-time_created") + ) + + print('SCRAPPED ISSUES') + print(issues) + + for issue in issues: + affected = issue.affected if isinstance(issue.affected, dict) else {} + if str(affected.get("id") or "") not in _ids: + continue + created = issue.time_created + age_days = max((now - created).days, 0) if created else None + records.append( + { + "title": issue.title or "Untitled issue", + "affected": affected.get("str"), + "created": created, + "age_days": age_days, + } + ) + + return {"count": len(records), "records": records} + + + def _build_tests_data(self, window_start): + tests_qs = ( + Test.objects.filter(site=self.site, time_completed__isnull=False, time_completed__gte=window_start) + .select_related("page") + .order_by("page_id", "-time_completed") + ) + + total_tests, pass_count, fail_count, incomplete_count = 0, 0, 0, 0 + score_total, score_count = 0.0, 0 + latest_by_page = {} + + for test in tests_qs: + total_tests += 1 + status_raw = (test.status or "").strip().lower() + if status_raw.startswith("pass"): + pass_count += 1 + elif status_raw.startswith("fail"): + fail_count += 1 + else: + incomplete_count += 1 + + if test.score is not None: + score_total += float(test.score) + score_count += 1 + + page_key = str(test.page_id) if test.page_id else f"__missing_{test.id}" + if page_key in latest_by_page: + continue + latest_by_page[page_key] = { + "page_url": test.page.page_url if test.page else "Unknown page", + "score": test.score, + "status": test.status or "unknown", + "time_completed": test.time_completed, + } + + return { + "rollup": { + "total_tests": total_tests, + "pass_count": pass_count, + "fail_count": fail_count, + "incomplete_count": incomplete_count, + "avg_score": round(score_total / score_count, 2) if score_count else None, }, - "f": { - "grade": "F", - "color": "#B43A29", + "pages": sorted(latest_by_page.values(), key=lambda x: (x["page_url"] or "")), + } + + + def _build_caseruns_data(self, window_start): + qs = CaseRun.objects.filter(site=self.site, time_created__gte=window_start).order_by("-time_created") + status_counts, latest_runs = {}, [] + for run in qs: + status_key = (run.status or "unknown").strip().lower() or "unknown" + status_counts[status_key] = status_counts.get(status_key, 0) + 1 + if len(latest_runs) < 10: + latest_runs.append({"title": run.title or "Untitled run", "status": run.status or "unknown", "time_created": run.time_created}) + return {"count": qs.count(), "status_counts": status_counts, "latest_runs": latest_runs} + + + def _build_performance_data(self, window_start): + scans_qs = ( + Scan.objects.filter(site=self.site, time_completed__isnull=False, time_completed__gte=window_start) + .select_related("page") + .order_by("page_id", "-time_completed") + ) + + latest_by_page, scores = {}, [] + for scan in scans_qs: + page_key = str(scan.page_id) if scan.page_id else f"__missing_{scan.id}" + if page_key in latest_by_page: + continue + if scan.score is not None: + scores.append(float(scan.score)) + latest_by_page[page_key] = {"page_url": scan.page.page_url if scan.page else "Unknown page", "health": scan.score, "time_completed": scan.time_completed} + + return { + "pages": sorted(latest_by_page.values(), key=lambda x: (x["page_url"] or "")), + "rollup": { + "avg_health": round(sum(scores) / len(scores), 2) if scores else None, + "min_health": round(min(scores), 2) if scores else None, + "max_health": round(max(scores), 2) if scores else None, + "pages_with_data": len(scores), }, + } + + def _build_datasets(self, selected_types, lookback_days): + now = timezone.now() + window_start = now - timedelta(days=lookback_days) + pages = list(Page.objects.filter(site=self.site).order_by("page_url")) + + datasets = {} + if "issues" in selected_types: + datasets["issues"] = self._build_issues_data(window_start=window_start, now=now) + if "tests" in selected_types: + datasets["tests"] = self._build_tests_data(window_start=window_start) + if "caseruns" in selected_types: + datasets["caseruns"] = self._build_caseruns_data(window_start=window_start) + if "performance" in selected_types: + datasets["performance"] = self._build_performance_data(window_start=window_start) + + return { + "generated_at": now, + "window_start": window_start, + "site": {"id": str(self.site.id), "site_url": self.site.site_url, "total_pages": len(pages)}, + "datasets": datasets, } - if score >= 80: - grade = score_types['a'] - elif 80 > score >= 70: - grade = score_types['b'] - elif 70 > score >= 50: - grade = score_types['c'] - elif 50 > score >= 30: - grade = score_types['d'] - elif 30 > score >= 0: - grade = score_types['e'] - else: - grade = score_types['f'] - - return grade - - - def get_cat_string(self, cat): - - if cat == 'fonts': - string = 'Fonts' - elif cat == 'badCSS': - string = 'Bad CSS' - elif cat == 'jQuery': - string = 'jQuery' - elif cat == 'requests': - string = 'Requests' - elif cat == 'pageWeight': - string = 'Page Weight' - elif cat == 'serverConfig': - string = 'Server Config' - elif cat == 'badJavascript': - string = 'Bad JS' - elif cat == 'cssComplexity': - string = 'CSS Complexity' - elif cat == 'domComplexity': - string = 'DOM Complexity' - elif cat == 'javascriptComplexity': - string = 'JS Complexity' - elif cat == 'seo': - string = 'SEO' - elif cat == 'pwa': - string = 'PWA' - elif cat == 'crux': - string = 'CRUX' - elif cat == 'best_practices' or cat == 'best-practices': - string = 'Best Practices' - elif cat == 'performance': - string = 'Performance' - elif cat == 'accessibility': - string = 'Accessibility' - - return string - - - - def create_data(self, data_type=str): + + def _safe_score(self, value): + if value is None: + return "n/a" + try: + return f"{float(value):.2f}" + except Exception: + return str(value) + + + def _render_cover_page(self, selected_types, lookback_days, snapshot): self.setup_page() - - if data_type == 'yellowlab': - data = self.scan.yellowlab - page_title = 'Yellow Lab' - avg_score = 'globalScore' - - if data_type == 'lighthouse': - data = self.scan.lighthouse - page_title = 'Lighthouse' - avg_score = 'average' - - self.draw_page_title(page_title) - if data['scores'][avg_score] is None: - return False - - # measurements - space = .25 - text_space = .05 - begin_y = 8 - log_margin = 3.7 - text_margin = .3 - value_margin = 3 - log_height = .2 - log_width = 4 - grade_tab_width = .07 - - c_count = 0 - logs_count = 0 - for cat in data['audits']: - - # checking if cat is not null - if data['scores'][cat] is not None: - - # creating global score - if c_count == 0: - grade_obj = self.get_score_data(data['scores'][avg_score]) - self.c.setFillColor(HexColor(grade_obj['color'],)) - self.c.roundRect( - 2*inch, - 8.7*inch, - 1*inch, - 1*inch, - .17*inch, - stroke=0, - fill=1 - ) - self.c.setFillColor(HexColor(self.text_color)) - self.c.setFont('Helvetica', 30) - self.c.drawCentredString( - 2.5*inch, - 9.05*inch, - grade_obj['grade'] - ) - self.c.setFont('Helvetica', 20) - self.c.drawCentredString( - 5.5*inch, - 8.9*inch, - 'Global Score' - ) - self.c.setFont('Helvetica-Bold', 20) - self.c.drawCentredString( - 5.5*inch, - 9.25*inch, - f'{data["scores"][avg_score]}/100' - ) - - - # creating new page at limit --> 20 items - if logs_count >= 20: - self.end_page() - logs_count = 0 - begin_y = 9 - self.setup_page() - self.draw_page_title(f'{page_title} (continued)') - - # creating space btw sections - if c_count > 0 and logs_count != 0: - begin_y = (self.y - .2) - - - - # creating individual grade cards - grade_obj = self.get_score_data(data['scores'][cat]) - self.c.setFillColor(HexColor(grade_obj['color'],)) - self.c.roundRect( - .5*inch, - (begin_y - .25)*inch, - .5*inch, - .5*inch, - .12*inch, - stroke=0, - fill=1 - ) - self.c.setFillColor(HexColor(self.text_color)) - self.c.setFont('Helvetica', 16) - self.c.drawCentredString( - .75*inch, - (begin_y - .07)*inch, - grade_obj['grade'] - ) - - self.c.setFont('Helvetica', 16) - cat_string = self.get_cat_string(cat) - self.c.drawCentredString( - 2.3*inch, - (begin_y - .07)*inch, - cat_string - ) - - - p_count = 0 - for policy in data['audits'][cat]: - - if (begin_y - (space * p_count)) < 1: - break - - # setting up keys for dict(s) - if data_type == 'yellowlab': - policy_text = policy["policy"]["label"] - policy_value = policy["value"] - binary = False - if data_type == 'lighthouse': - policy_text = policy["title"] - policy_value = '' - if "displayValue" in policy: - if len(policy["displayValue"]) < 9: - policy_value = policy["displayValue"] - binary = True - - - if len(policy_text) < 53: - # creating log box - self.c.setFont('Helvetica', 9) - self.c.setFillColor(HexColor(f'{self.highlight_color}95', hasAlpha=True)) - self.c.rect( - log_margin*inch, - (begin_y - (space * p_count))*inch, - log_width*inch, log_height*inch, - stroke=0, - fill=1 - ) - - # get grade tab - grade_obj = self.get_score_data(policy['score'], is_binary=binary) - self.c.setFillColor(HexColor(grade_obj['color'],)) - self.c.rect( - log_margin*inch, - (begin_y - (space * p_count))*inch, - grade_tab_width*inch, - log_height*inch, - stroke=0, - fill=1 - ) - - # inserting data - self.c.setFillColor(HexColor(self.text_color)) - - # text - self.c.drawString( - (log_margin + text_margin)*inch, - ((begin_y - (space * p_count)) + text_space)*inch, - (f'{policy_text}') - ) - - # value - self.c.drawString( - (value_margin + text_margin + log_margin)*inch, - ((begin_y - (space * p_count)) + text_space)*inch, - (f'{policy_value}') - ) - - - p_count += 1 - logs_count += 1 - self.y = (begin_y - (space * p_count)) - - c_count += 1 - - - + now = snapshot["generated_at"] + window_start = snapshot["window_start"] + date_range = f"{window_start.strftime('%b %d, %Y')} - {now.strftime('%b %d, %Y')}" + + self.c.setFillColor(HexColor(self.text_color)) + self.c.setFont(self.FONT_BOLD, 40) + self.c.drawString(0.5 * inch, 9.65 * inch, "Site Report") + + self.c.setFont(self.FONT_BOLD, 18) + self.draw_wrapped_line(text=self.site.site_url or f"Site {self.site.id}", length=58, x_pos=0.5, y_pos=9.15, y_offset=0.3) + + self.c.setFont(self.FONT_REGULAR, 10) + # self.c.drawString(0.5 * inch, 8.3 * inch, f"Generated: {now.strftime('%Y-%m-%d %H:%M %Z')}") + # self.c.drawString(0.5 * inch, 8.1 * inch, f"Report range: {date_range}") + # self.c.drawString(0.5 * inch, 7.9 * inch, f"Sections: {', '.join(selected_types)}") + + self._draw_stat_card(0.5, 6.9, 2.35, 1.0, "Site Pages", str(snapshot["site"]["total_pages"])) + self._draw_stat_card(3.0, 6.9, 2.0, 1.0, "Sections", str(len(selected_types))) + self._draw_stat_card(5.15, 6.9, 2.85, 1.0, "Date Range", date_range, value_font_size=9) + + section_items = [{"headline": s.title(), "meta": f"Included in this export ({lookback_days}d window)", "badge": "enabled", "badge_color": "#38B43F", "accent_color": "#4B79DB"} for s in selected_types] + self._draw_design_list(0.5, 6.5, 7.5, 0.5, "Included sections", section_items, max_rows=8) self.end_page() + def _render_issues_section(self, data): + self.setup_page() + self.draw_page_title("Issues", "Open issues created in the selected lookback window") + + records = data.get("records", []) + ages = [int(item.get("age_days") or 0) for item in records] + avg_age = round(sum(ages) / len(ages), 1) if ages else 0 + max_age = max(ages) if ages else 0 + + buckets = {"0-1d": 0, "2-7d": 0, "8-30d": 0, "30+d": 0} + for age in ages: + if age <= 1: + buckets["0-1d"] += 1 + elif age <= 7: + buckets["2-7d"] += 1 + elif age <= 30: + buckets["8-30d"] += 1 + else: + buckets["30+d"] += 1 + + self._draw_stat_card(0.5, 8.25, 2.45, 1.0, "Open Issues", str(data.get("count", 0))) + self._draw_stat_card(3.1, 8.25, 2.45, 1.0, "Avg Age", f"{avg_age}d") + self._draw_stat_card(5.7, 8.25, 2.3, 1.0, "Oldest", f"{max_age}d") + + self._draw_bar_chart(0.5, 5.25, 3.9, 2.7, "Issue age buckets", list(buckets.keys()), [float(v) for v in buckets.values()]) + + items = [] + for issue in records[:8]: + created_str = issue["created"].strftime("%Y-%m-%d") if issue.get("created") else "n/a" + age_days = int(issue.get("age_days", 0) or 0) + badge_color = "#38B43F" if age_days <= 1 else ("#E3A635" if age_days <= 7 else "#DB524B") + items.append({"headline": issue.get("title", "Untitled issue"), "meta": f"affected: {issue.get('affected', 'site')} | created: {created_str}", "badge": f"{age_days}d", "badge_color": badge_color, "accent_color": badge_color}) + + if not items: + items = [{"headline": "No open site issues were created during this lookback window.", "meta": "Everything in the selected window looks clean.", "badge": "ok", "badge_color": "#38B43F", "accent_color": "#38B43F"}] + + self._draw_design_list(4.55, 7.95, 3.45, 0.5, "Recent open issues", items, max_rows=8) + self.end_page() + def _render_tests_section(self, data): + self.setup_page() + self.draw_page_title("Tests", "Latest completed tests per page with site-level status mix") + rollup = data.get("rollup", {}) + pages = data.get("pages", []) + self._draw_stat_card(0.5, 8.25, 1.85, 1.0, "Total", str(rollup.get("total_tests", 0))) + self._draw_stat_card(2.45, 8.25, 1.85, 1.0, "Pass", str(rollup.get("pass_count", 0))) + self._draw_stat_card(4.4, 8.25, 1.85, 1.0, "Fail", str(rollup.get("fail_count", 0))) + self._draw_stat_card(6.35, 8.25, 1.65, 1.0, "Avg Score", self._safe_score(rollup.get("avg_score"))) + self._draw_pie_chart(0.5, 5.15, 3.9, 2.9, "Status distribution", [("pass", rollup.get("pass_count", 0)), ("fail", rollup.get("fail_count", 0)), ("incomplete", rollup.get("incomplete_count", 0))]) + score_rows = [item for item in pages if item.get("score") is not None][:8] + labels = [item.get("page_url", "page") for item in score_rows] + values = [float(item.get("score") or 0) for item in score_rows] + self._draw_bar_chart(4.55, 5.15, 3.45, 2.9, "Per-page latest score", labels, values) + items = [] + for item in pages[:7]: + completed = item["time_completed"].strftime("%Y-%m-%d") if item.get("time_completed") else "n/a" + status_label = str(item.get("status", "unknown")).lower() + badge_color = "#38B43F" if status_label.startswith("pass") else ("#DB524B" if status_label.startswith("fail") else "#E3A635") + items.append({"headline": item.get("page_url", "Unknown page"), "meta": f"score: {self._safe_score(item.get('score'))} | completed: {completed}", "badge": item.get("status", "unknown"), "badge_color": badge_color, "accent_color": badge_color}) + if not items: + items = [{"headline": "No completed tests found in this lookback window.", "meta": "Run tests to populate this section.", "badge": "none", "badge_color": "#4B79DB", "accent_color": "#4B79DB"}] + self._draw_design_list(0.5, 4.8, 7.5, 0.5, "Per-page latest completed tests", items, max_rows=7) + self.end_page() + def _render_caseruns_section(self, data): + self.setup_page() + self.draw_page_title("Case Runs", "Case run activity and status distribution in lookback window") + status_counts = data.get("status_counts", {}) + latest_runs = data.get("latest_runs", []) + self._draw_stat_card(0.5, 8.25, 2.45, 1.0, "Total Runs", str(data.get("count", 0))) + self._draw_stat_card(3.1, 8.25, 2.45, 1.0, "Statuses", str(len(status_counts.keys()))) + self._draw_stat_card(5.7, 8.25, 2.3, 1.0, "Latest Rows", str(len(latest_runs))) + ordered_statuses = sorted(status_counts.items(), key=lambda i: i[1], reverse=True) + self._draw_pie_chart(0.5, 5.2, 3.9, 2.8, "Run statuses", [(k, v) for k, v in ordered_statuses]) + labels = [item[0] for item in ordered_statuses[:8]] + values = [float(item[1]) for item in ordered_statuses[:8]] + self._draw_bar_chart(4.55, 5.2, 3.45, 2.8, "Status counts", labels, values) + items = [] + for run in latest_runs[:7]: + created = run["time_created"].strftime("%Y-%m-%d %H:%M") if run.get("time_created") else "n/a" + status_label = str(run.get("status", "unknown")).lower() + badge_color = "#38B43F" if status_label in ["passed", "pass", "success", "complete"] else ("#DB524B" if status_label in ["failed", "fail", "error"] else "#E3A635") + items.append({"headline": run.get("title", "Untitled run"), "meta": f"created: {created}", "badge": run.get("status", "unknown"), "badge_color": badge_color, "accent_color": badge_color}) + if not items: + items = [{"headline": "No case runs found in this lookback window.", "meta": "Run a case to populate this section.", "badge": "none", "badge_color": "#4B79DB", "accent_color": "#4B79DB"}] + self._draw_design_list(0.5, 4.8, 7.5, 0.5, "Latest runs", items, max_rows=7) + self.end_page() + def _render_performance_section(self, data): + self.setup_page() + self.draw_page_title("Performance", "Latest completed scan health scores per page") + rollup = data.get("rollup", {}) + pages = data.get("pages", []) + self._draw_stat_card(0.5, 8.25, 1.85, 1.0, "Pages", str(rollup.get("pages_with_data", 0))) + self._draw_stat_card(2.45, 8.25, 1.85, 1.0, "Avg", self._safe_score(rollup.get("avg_health"))) + self._draw_stat_card(4.4, 8.25, 1.85, 1.0, "Min", self._safe_score(rollup.get("min_health"))) + self._draw_stat_card(6.35, 8.25, 1.65, 1.0, "Max", self._safe_score(rollup.get("max_health"))) + scored_pages = [item for item in pages if item.get("health") is not None] + labels = [item.get("page_url", "page") for item in scored_pages[:10]] + values = [float(item.get("health") or 0) for item in scored_pages[:10]] + self._draw_bar_chart(0.5, 5.2, 7.5, 2.8, "Latest page health scores", labels, values) + lowest = sorted(scored_pages, key=lambda x: float(x.get("health") or 0))[:7] + items = [] + for item in lowest: + completed = item["time_completed"].strftime("%Y-%m-%d") if item.get("time_completed") else "n/a" + score = float(item.get("health") or 0) + badge_color = "#38B43F" if score >= 80 else ("#E3A635" if score >= 50 else "#DB524B") + items.append({"headline": item.get("page_url", "Unknown page"), "meta": f"completed: {completed}", "badge": self._safe_score(item.get("health")), "badge_color": badge_color, "accent_color": badge_color}) - def make_test_report(self): - - self.cover_page() - - if 'lighthouse' in self.report.type or 'full' in self.report.type: - self.create_data(data_type='lighthouse') + if not items: + items = [{"headline": "No completed scans found in this lookback window.", "meta": "Run scans to populate performance health rows.", "badge": "none", "badge_color": "#4B79DB", "accent_color": "#4B79DB"}] - if 'yellowlab' in self.report.type or 'full' in self.report.type: - self.create_data(data_type='yellowlab') + self._draw_design_list(0.5, 4.8, 7.5, 0.5, "Lowest health pages (attention)", items, max_rows=7) + self.end_page() - if 'crux' in self.report.type or 'full' in self.report.type: - self.setup_page() - self.draw_page_title('CRUX') - self.end_page() - self.publish_report() - return self.report + def _render_sections(self, selected_types, snapshot): + datasets = snapshot["datasets"] + for section_type in selected_types: + if section_type == "issues": + self._render_issues_section(datasets.get("issues", {"count": 0, "records": []})) + elif section_type == "tests": + self._render_tests_section(datasets.get("tests", {"rollup": {"total_tests": 0, "pass_count": 0, "fail_count": 0, "incomplete_count": 0, "avg_score": None}, "pages": []})) + elif section_type == "caseruns": + self._render_caseruns_section(datasets.get("caseruns", {"count": 0, "status_counts": {}, "latest_runs": []})) + elif section_type == "performance": + self._render_performance_section(datasets.get("performance", {"rollup": {"avg_health": None, "min_health": None, "max_health": None, "pages_with_data": 0}, "pages": []})) - + def publish_report(self) -> None: + self.c.save() + remote_path = f"static/sites/{self.site.id}/reports/{self.report.id}.pdf" + + with open(self.local_path, "rb") as data: + self.s3.upload_fileobj( + data, + str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, + ExtraArgs={"ACL": "public-read", "ContentType": "application/pdf"}, + ) + + report_url = f"{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0" + self.report.path = report_url + self.report.save() + os.remove(self.local_path) + def generate_report(self) -> dict: + message = "Report generation failed" + success = False + + site, site_error = self._resolve_site() + if site_error: + return {"report": self.report, "success": success, "message": site_error} + + lookback_days, lookback_error = self._get_lookback_days() + if lookback_error: + return {"report": self.report, "success": success, "message": lookback_error} + + selected_types, type_error = self._normalize_types() + if type_error: + return {"report": self.report, "success": success, "message": type_error} + + snapshot = self._build_datasets(selected_types=selected_types, lookback_days=lookback_days) + + info = self.report.info if isinstance(self.report.info, dict) else {} + info.update( + { + "text_color": self.text_color, + "highlight_color": self.highlight_color, + "background_color": self.background_color, + "lookback_days": lookback_days, + "types": selected_types, + "snapshot": { + "generated_at": snapshot["generated_at"].isoformat(), + "window_start": snapshot["window_start"].isoformat(), + "site": snapshot["site"], + "counts": { + "issues": snapshot["datasets"].get("issues", {}).get("count", 0), + "tests_pages": len(snapshot["datasets"].get("tests", {}).get("pages", [])), + "tests_total": snapshot["datasets"].get("tests", {}).get("rollup", {}).get("total_tests", 0), + "caseruns": snapshot["datasets"].get("caseruns", {}).get("count", 0), + "performance_pages": len(snapshot["datasets"].get("performance", {}).get("pages", [])), + }, + }, + } + ) + + self.report.info = info + self.report.type = selected_types + + self._render_cover_page(selected_types=selected_types, lookback_days=lookback_days, snapshot=snapshot) + self._render_sections(selected_types=selected_types, snapshot=snapshot) + + self.publish_report() + message = "Report Generated" + success = True - \ No newline at end of file + return {"report": self.report, "success": success, "message": message} diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py index 13424f82..ad4c9652 100644 --- a/app/api/utils/scanner.py +++ b/app/api/utils/scanner.py @@ -1,104 +1,110 @@ -from .driver_s import driver_init as driver_s_init, quit_driver -from .driver_s import driver_wait -from .driver_p import get_data -from ..models import Site, Scan, Test -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder +from .driver import ( + driver_init, quit_driver, + driver_wait , get_data +) +from ..models import * +from .alerter import Alerter from .lighthouse import Lighthouse from .yellowlab import Yellowlab -from .image import Image +from .imager import Imager +from .updater import update_flowrun +from .manager import record_task +from .tester import Tester +from django.core.cache import cache from datetime import datetime -import time, os, sys, json, asyncio +from cursion import settings +import os, asyncio, uuid, boto3, random, time + + + class Scanner(): + """ + Used to run and build all the + components of a new `Scan` + + Expects -> { + 'site' : object, + 'page' : object, + 'scan' : object, + 'type' : list + } + + Use self.build_scan() to create a new Scan + + Returns: `Scan` object + """ + + + def __init__( self, - site=None, - scan=None, - configs=None, - type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + site: object=None, + page: object=None, + scan: object=None, + type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] ): - if site == None and scan != None: - site = scan.site - - if configs is None: - configs = { - 'window_size': '1920,1080', - 'driver': 'selenium', - 'device': 'desktop', - 'mask_ids': None, - 'interval': 5, - 'min_wait_time': 10, - 'max_wait_time': 60, - 'timeout': 300, - 'disable_animations': False - } - self.site = site - - if configs['driver'] == 'selenium': - self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device']) - - if scan is not None: - self.scan = scan - else: - self.scan = None - - self.configs = configs + self.page = page + self.scan = scan self.type = type + # getting page and site if None + if site == None and scan != None: + self.site = scan.site + if page == None and scan != None: + self.page = scan.page + + - def first_scan(self): + def build_scan(self) -> object: """ - Method to run a scan independently of an existing `scan` obj. + Method to run a scan independently of an existing `scan` obj. - returns -> `Scan` + Returns: `Scan` """ + # setting defaults html = None logs = None images = None lh_data = None yl_data = None - - if self.scan is None: - self.scan = Scan.objects.create(site=self.site, type=self.type) - - if self.configs['driver'] == 'selenium': - self.driver.get(self.site.site_url) - if 'html' in self.scan.type or 'full' in self.scan.type: - html = self.driver.page_source - if 'logs' in self.scan.type or 'full' in self.scan.type: - logs = self.driver.get_log('browser') - if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - quit_driver(self.driver) - else: - driver_data = asyncio.run( - get_data( - url=self.site.site_url, - configs=self.configs - ) - ) - if 'html' in self.scan.type or 'full' in self.scan.type: - html = driver_data['html'] - if 'logs' in self.scan.type or 'full' in self.scan.type: - logs = driver_data['logs'] - if 'vrt' in self.scan.type or 'full' in self.scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + # running scan steps with selenium driver + driver = driver_init( + browser=self.scan.configs.get('browser', 'chrome'), + window_size=self.scan.configs['window_size'], + device=self.scan.configs['device'] + ) + driver.get(self.page.page_url) + driver_data = get_data( + driver=driver, + browser=self.scan.configs.get('browser', 'chrome'), + max_wait_time=self.scan.configs['max_wait_time'] + ) + if 'html' in self.scan.type or 'full' in self.scan.type: + html = driver_data['html'] + if 'logs' in self.scan.type or 'full' in self.scan.type: + logs = driver_data['logs'] + if 'vrt' in self.scan.type or 'full' in self.scan.type: + images = Imager(scan=self.scan).scan_vrt(driver=driver) if 'lighthouse' in self.scan.type or 'full' in self.scan.type: - lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() + lh_data = Lighthouse(scan=self.scan).get_data() if 'yellowlab' in self.scan.type or 'full' in self.scan.type: - yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() + yl_data = Yellowlab(scan=self.scan).get_data() + + # quiting selenium instance + quit_driver(driver) + # updating Scan object if html is not None: - self.scan.html = html + save_html(html, self.scan) if logs is not None: self.scan.logs = logs if images is not None: @@ -108,174 +114,203 @@ def first_scan(self): if yl_data is not None: self.scan.yellowlab = yl_data - self.scan.configs = self.configs + # saving scan data self.scan.time_completed = datetime.now() self.scan.save() - first_scan = self.scan - update_site_info(first_scan) + # update Scan.score + update_scan_score(self.scan) - return first_scan + # updating Site and Page objects + update_page_info(self.scan) + update_site_info(self.scan) + # return updated scan obj + return self.scan - def second_scan(self): - """ - Method to run a scan and attach existing `Scan` obj to it. +def update_scan_score(scan: object) -> object: + """ + Method to calculate the average health score and update + for the passed scan - returns -> `Scan` - """ - if not self.scan: - first_scan = Scan.objects.filter( - site=self.site, - time_completed__isnull=False - ).order_by('-time_created').first() - - else: - first_scan = self.scan + Args: + 'scan': object + + Returns: `Scan` + """ + + # setting defaults + score = None + scores = [] - # create second scan obj - second_scan = Scan.objects.create(site=self.site, type=self.type) + # get latest scan scores + if scan.lighthouse['scores']['average'] is not None: + scores.append(scan.lighthouse['scores']['average']) + if scan.yellowlab['scores']['globalScore'] is not None: + scores.append(scan.yellowlab['scores']['globalScore']) + + # calc average score + if len(scores) > 0: + score = sum(scores)/len(scores) - html = None - logs = None - images = None - lh_data = None - yl_data = None - - if self.configs['driver'] == 'selenium': - self.driver.get(self.site.site_url) - if 'html' in second_scan.type or 'full' in second_scan.type: - html = self.driver.page_source - if 'logs' in second_scan.type or 'full' in second_scan.type: - logs = self.driver.get_log('browser') - if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = Image().scan(site=self.site, driver=self.driver, configs=self.configs) - quit_driver(self.driver) - else: - driver_data = asyncio.run( - get_data( - url=self.site.site_url, - configs=self.configs - ) - ) - if 'html' in second_scan.type or 'full' in second_scan.type: - html = driver_data['html'] - if 'logs' in second_scan.type or 'full' in second_scan.type: - logs = driver_data['logs'] - if 'vrt' in second_scan.type or 'full' in second_scan.type: - images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs)) + # save to scan + scan.score = score + scan.save() - if 'lighthouse' in second_scan.type or 'full' in second_scan.type: - lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() - if 'yellowlab' in second_scan.type or 'full' in second_scan.type: - yl_data = Yellowlab(site=self.site, configs=self.configs).get_data() - - if html is not None: - second_scan.html = html - if logs is not None: - second_scan.logs = logs - if images is not None: - second_scan.images = images - if lh_data is not None: - second_scan.lighthouse = lh_data - if yl_data is not None: - second_scan.yellowlab = yl_data + # returning scan + return scan - second_scan.configs = self.configs - second_scan.time_completed = datetime.now() - second_scan.paired_scan = first_scan - second_scan.save() - - first_scan.paried_scan = second_scan - first_scan.save() - update_site_info(second_scan) - - return second_scan +def update_site_info(scan: object) -> object: + """ + Method to update associated Site with the new Scan data + Args: + 'scan': object + + Returns: `Site` + """ + + # setting defaults + score = None + scores = [] + site = scan.site + pages = Page.objects.filter(site=site) + + # get latest scan of pages + scans = [] + for page in pages: + if Scan.objects.filter(page=page).exists(): + scan = Scan.objects.filter(page=page).order_by('-time_completed')[0] + if scan.score: + scores.append(scan.score) + + # calc average score + if len(scores) > 0: + score = sum(scores)/len(scores) + + # saving new info to site + site.info['latest_scan']['id'] = str(scan.id) + site.info['latest_scan']['time_created'] = str(scan.time_created) + site.info['latest_scan']['time_completed'] = str(scan.time_completed) + site.info['latest_scan']['score'] = score + site.save() + # returning site + return site -def update_site_info(scan): +def update_page_info(scan: object) -> object: """ - Method to update associated Site with the new Scan data + Method to update associated Page with the new Scan data - returns -> `Site` + Args: + 'scan': object + + Returns: `Page` """ - - health = 'No Data' - badge = 'neutral' - d = 0 - score = 0 - site = scan.site - if scan.lighthouse['scores']['average'] is not None: - score += float(scan.lighthouse['scores']['average']) - d += 1 - if scan.yellowlab['scores']['globalScore'] is not None: - score += float(scan.yellowlab['scores']['globalScore']) - d += 1 - - if score != 0: - score = score / d - if score >= 75: - health = 'Good' - badge = 'success' - elif 75 > score >= 60: - health = 'Okay' - badge = 'warning' - elif 60 > score: - health = 'Poor' - badge = 'danger' - - else: - if scan.site.info['status']['score'] is not None: - score = float(site.info['status']['score']) - health = site.info['status']['health'] - badge = site.info['status']['badge'] - else: - score = None + # saving new info to page + scan.page.info['latest_scan']['id'] = str(scan.id) + scan.page.info['latest_scan']['time_created'] = str(scan.time_created) + scan.page.info['latest_scan']['time_completed'] = str(scan.time_completed) + scan.page.info['latest_scan']['score'] = scan.score + scan.page.save() - site.info['latest_scan']['id'] = str(scan.id) - site.info['latest_scan']['time_created'] = str(scan.time_created) - site.info['latest_scan']['time_completed'] = str(scan.time_completed) - site.info['lighthouse'] = scan.lighthouse.get('scores') - site.info['yellowlab'] = scan.yellowlab.get('scores') - site.info['status']['health'] = str(health) - site.info['status']['badge'] = str(badge) - site.info['status']['score'] = score + # returning page + return scan.page - site.save() - return site +def save_html(html: str, scan: object) -> object: + """ + Saves html page source as a '.txt' file and uploads + to s3. Then saves the remote uri to the `scan` obj. + Args: + html: str, + scan: object + + Returns: `Scan` + """ + # setup boto3 configuration + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # save html data as text file + file_id = uuid.uuid4() + with open(f'{file_id}.txt', 'w') as fp: + fp.write(html) + + # upload to s3 and return url + html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt') + remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt' + root_path = settings.AWS_S3_URL_PATH + html_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(html_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"} + ) + + # save to scan obj + scan.html = html_url + scan.save() + # remove local copy + os.remove(html_file) + # return scan + return scan -def check_scan_completion(scan): +def check_scan_completion( + scan: object, + sender: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> object: """ - Method that checks if the scan has finished all - components. If so, method also updates scan and site - info. - - returns -> `Scan` + Method that checks if the scan has finished all + components. If so, method also updates Scan, Site, + & Page info. + + Args: + scan: object, + sender: str, + test_id: str, + alert_id: str + flowrun_id: str, + node_index: str + + Returns: `Scan` """ + # sleeping random for DB update + time.sleep(random.uniform(0.1, 2)) + + # setting defaults finished = True + # checking for each scan type completion if 'html' in scan.type or 'full' in scan.type: if scan.html == None or scan.html == '': finished = False @@ -298,151 +333,457 @@ def check_scan_completion(scan): # deciding if done if finished is True: - time_completed = datetime.now() - update_site_info(scan) - scan.time_completed = time_completed + scan.time_completed = datetime.now() scan.save() - return scan + # update assoc site, page, & scan score + update_scan_score(scan) + update_page_info(scan) + update_site_info(scan) + # resolve stable tracking identity when + # this scan is part of a FlowRun + track_id = str(scan.id) + source_id = str(scan.id) + if test_id is not None: + source_id = str(test_id) + track_id = str(test_id) + try: + test_obj = Test.objects.get(id=test_id) + first_task = ((test_obj.system or {}).get('tasks') or [{}])[0] + task_kwargs = first_task.get('kwargs') or {} + track_id = str(task_kwargs.get('track_id') or test_id) + except Exception: + pass + + # add scan to objects + objects = [{ + 'parent': str(scan.page.id), + 'id': str(test_id) if test_id else str(scan.id), + 'source_id': source_id, + 'track_id': track_id, + 'status': 'working' if test_id else 'passed' + }] + + # update flowrun + if flowrun_id and flowrun_id != 'None': + time.sleep(random.uniform(0.1, 5)) + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': f'finished running all scan components for {scan.page.page_url} | scan_id: {str(scan.id)}', + 'objects': objects + }) + + # start Test if test_id present + if test_id is not None: + + # derive tracking identity from Test system data + run_track_id = str(test_id) + try: + test = Test.objects.get(id=test_id) + first_task = ((test.system or {}).get('tasks') or [{}])[0] + task_kwargs = first_task.get('kwargs') or {} + run_track_id = str(task_kwargs.get('track_id') or test_id) + except Exception: + test = None + + # if Test already completed, do not re-run it. + if test is not None and test.time_completed is not None: + if flowrun_id and flowrun_id != 'None': + objects[-1]['status'] = test.status + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': f'skipping run_test for completed test_id: {str(test_id)}', + 'objects': objects + }) + return scan + + # avoid duplicate test launches from concurrent component completions + run_test_lock_key = f'scanner:run_test:{str(test_id)}' + if not cache.add(run_test_lock_key, '1', timeout=600): + return scan + + try: + # update flowrun + if flowrun_id and flowrun_id != 'None': + time.sleep(random.uniform(0.1, 5)) + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': f'starting test comparison algorithm for {scan.page.page_url} | test_id: {str(test_id)}', + 'objects': objects + }) + + # get task_id from scan.system + task_id = None + for task in scan.system['tasks']: + if task.get('component') == sender: + task_id = task.get('task_id') + + # record task data in test + record_task( + resource_type='test', + resource_id=str(test_id), + task_id=str(task_id), + task_method='run_test', + kwargs={ + 'test_id': str(test_id), + 'alert_id': str(alert_id) if alert_id is not None else None, + 'flowrun_id': str(flowrun_id) if flowrun_id is not None else None, + 'node_index': str(node_index) if node_index is not None else None, + 'track_id': run_track_id + } + ) + + print('\n---------------\nScan Complete\nStarting Test...\n---------------\n') + test = Test.objects.get(id=test_id) + updated_test = Tester(test=test).run_test() + + # update flowrun + if flowrun_id and flowrun_id != 'None': + objects[-1]['status'] = updated_test.status + update_flowrun(**{ + 'flowrun_id': str(flowrun_id), + 'node_index': node_index, + 'message': ( + f'test for {scan.page.page_url} completed with status: '+ + f'{"❌ FAILED" if updated_test.status == 'failed' else "✅ PASSED"} | test_id: {str(test_id)}' + ), + 'objects': objects + }) + finally: + cache.delete(run_test_lock_key) + + if alert_id is not None and alert_id != 'None': + print('running alert from `cursion.check_scan_completion`') + obj_id = test_id if test_id else str(scan.id) + Alerter(alert_id=alert_id, object_id=obj_id).run_alert() + # returning scan + return scan -def _html_and_logs(scan_id): +def _html_and_logs( + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> object: """ - Method to run the 'html' and 'logs' component of the scan - allowing for multi-threading. - - returns -> `Scan` + Method to run the 'html' and 'logs' component of the scan + allowing for multi-threading. + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str + + Returns: `Scan` """ - scan = Scan.objects.get(id=scan_id) - if scan.configs['driver'] == 'selenium': + # retrieve scan + scan = Scan.objects.get(id=scan_id) - driver = driver_s_init( + # setting defaults + message = None + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': f'starting html and logs component for {scan.page.page_url} | scan_id: {scan_id}', + }) + + try: + # get html and logs using selenium + # init driver & get data + driver = driver_init( + browser=scan.configs.get('browser', 'chrome'), window_size=scan.configs['window_size'], device=scan.configs['device'] ) - driver.get(scan.site.site_url) - if 'html' in scan.type or 'full' in scan.type: - html = driver.page_source - scan = Scan.objects.get(id=scan_id) - scan.html = html - scan.save() - if 'logs' in scan.type or 'full' in scan.type: - logs = driver.get_log('browser') - scan = Scan.objects.get(id=scan_id) - scan.logs = logs - scan.save() - quit_driver(driver) - - - if scan.configs['driver'] == 'puppeteer': - - driver_data = asyncio.run( - get_data( - url=scan.site.site_url, - configs=scan.configs - ) + driver.get(scan.page.page_url) + driver_data = get_data( + driver=driver, + browser=scan.configs.get('browser', 'chrome'), + max_wait_time=int(scan.configs['max_wait_time']), + min_wait_time=int(scan.configs['min_wait_time']), + interval=int(scan.configs['interval']) ) if 'html' in scan.type or 'full' in scan.type: html = driver_data['html'] scan = Scan.objects.get(id=scan_id) - scan.html = html - scan.save() + save_html(html, scan) if 'logs' in scan.type or 'full' in scan.type: logs = driver_data['logs'] scan = Scan.objects.get(id=scan_id) scan.logs = logs scan.save() + quit_driver(driver) + + # setting flowrun log + message = f'completed html and logs component for {scan.page.page_url} | scan_id: {scan_id}' + except Exception as e: + print(e) + + # setting flowrun log + message = f'html and logs component failed for {scan.page.page_url} | scan_id: {scan_id}' + + # try to quit selenium session + try: + quit_driver(driver) + except: + pass + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': message, + }) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, 'html', test_id, alert_id, flowrun_id, node_index) + # return udpated scan return scan - -def _vrt(scan_id): +def _vrt( + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> object: """ - Method to run the visual regression (vrt) component of the scan - allowing for multi-threading. - - returns -> `Scan` + Method to run the visual regression (vrt) component of the scan + allowing for multi-threading. + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str + + Returns: `Scan` """ + + # retrieve scan scan = Scan.objects.get(id=scan_id) - if scan.configs['driver'] == 'selenium': - driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device']) - images = Image().scan(site=scan.site, driver=driver, configs=scan.configs) + + # setting defaults + message = None + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': f'starting images (vrt) component for {scan.page.page_url} | scan_id: {scan_id}', + }) + + try: + # run Imager using selenium + driver = driver_init( + window_size=scan.configs.get('window_size', '1920,1080'), + device=scan.configs.get('device', 'desktop'), + browser=scan.configs.get('browser', 'chrome') + ) + images = Imager(scan=scan).scan_vrt(driver=driver) quit_driver(driver) + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.images = images + scan.save() - if scan.configs['driver'] == 'puppeteer': - images = asyncio.run(Image().scan_p(site=scan.site, configs=scan.configs)) + # setting flowrun log + message = f'completed images (vrt) component for {scan.page.page_url} | scan_id: {scan_id}' + + except Exception as e: + print(e) + + # setting flowrun log + message = f'html and logs component failed for {scan.page.page_url} | scan_id: {scan_id}' - # updating Scan object - scan = Scan.objects.get(id=scan_id) - scan.images = images - scan.save() + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': message + }) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, 'vrt', test_id, alert_id, flowrun_id, node_index) + # returning updated scan return scan - -def _lighthouse(scan_id): +def _lighthouse( + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> object: """ - Method to run the lighthouse component of the scan - allowing for multi-threading. - - returns -> `Scan` + Method to run the lighthouse component of the scan + allowing for multi-threading. + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str + + Returns: `Scan` """ - scan = Scan.objects.get(id=scan_id) - # running lighthouse - lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() - - # updating Scan object + # retrieve scan scan = Scan.objects.get(id=scan_id) - scan.lighthouse = lh_data - scan.save() - # checking if scan is done - scan = check_scan_completion(scan) + # setting defaults + message = None + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': f'starting lighthouse component for {scan.page.page_url} | scan_id: {scan_id}', + }) + + try: + # running lighthouse + lh_data = Lighthouse(scan=scan).get_data() + print(f'LIGHTHOUSE failure_status -> {lh_data.get('failed')}') + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.lighthouse = lh_data + scan.save() - return scan + # setting flowrun log + message = f'completed lighthouse component for {scan.page.page_url} | scan_id: {scan_id}' + + except Exception as e: + scan.lighthouse['failed'] = True + scan.save() + print(e) + # setting flowrun log + message = f'lighthouse component failed for {scan.page.page_url} | scan_id: {scan_id}' + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': message + }) + # update scan score + update_scan_score(scan) + # checking if scan is done + scan = check_scan_completion(scan, 'lighthouse', test_id, alert_id, flowrun_id, node_index) + + # returning updated scan + return scan -def _yellowlab(scan_id): - """ - Method to run the yellowlab component of the scan - allowing for multi-threading. - returns -> `Scan` + +def _yellowlab( + scan_id: str=None, + test_id: str=None, + alert_id: str=None, + flowrun_id: str=None, + node_index: str=None + ) -> object: + """ + Method to run the yellowlab component of the scan + allowing for multi-threading. + + Args: + scan_id : str, + test_id : str, + alert_id : str, + flowrun_id : str, + node_index : str + + Returns: `Scan` """ + + # retrieve scan scan = Scan.objects.get(id=scan_id) - # running yellowlab - yl_data = Yellowlab(site=scan.site, configs=scan.configs).get_data() + # setting defaults + message = None + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': f'starting yellowlab component for {scan.page.page_url} | scan_id: {scan_id}', + }) - # updating Scan object - scan = Scan.objects.get(id=scan_id) - scan.yellowlab = yl_data - scan.save() + try: + # running yellowlab + yl_data = Yellowlab(scan=scan).get_data() + print(f'YELLOWLAB failure_status -> {yl_data.get('failed')}') + + # updating Scan object + scan = Scan.objects.get(id=scan_id) + scan.yellowlab = yl_data + scan.save() + + # setting flowrun log + message = f'completed yellowlab component for {scan.page.page_url} | scan_id: {scan_id}' + + except Exception as e: + scan.yellowlab['failed'] = True + scan.save() + print(e) + + # setting flowrun log + message = f'yellowlab component failed for {scan.page.page_url} | scan_id: {scan_id}' + + # update flowrun + if flowrun_id and flowrun_id != 'None': + update_flowrun(**{ + 'flowrun_id': flowrun_id, + 'node_index': node_index, + 'message': message + }) + + # update scan score + update_scan_score(scan) # checking if scan is done - scan = check_scan_completion(scan) + scan = check_scan_completion(scan, 'yellowlab', test_id, alert_id, flowrun_id, node_index) + # returning updated scan return scan + + diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py index d5dd0d4a..54c87ef3 100644 --- a/app/api/utils/tester.py +++ b/app/api/utils/tester.py @@ -1,14 +1,37 @@ -from ..models import Site, Scan, Test -import time, os, sys, json, random, string, re -from difflib import SequenceMatcher, HtmlDiff, Differ +from ..models import * from datetime import datetime -from .image import Image +from .imager import Imager +from cursion import settings +from difflib import SequenceMatcher +from .issuer import Issuer +import os, json, random, \ +string, re, requests, uuid, boto3 + + + class Tester(): + """ + Used to run and build all the + components of a new `Test` + + Expects -> { + 'test' : object, + } + + Use self.run_test() to run all Test components + + Returns: `Test` object + """ + + + - def __init__(self, test): + def __init__(self, test: object): + + # setting defaults self.test = test self.pre_scan_html = [] self.post_scan_html = [] @@ -17,26 +40,45 @@ def __init__(self, test): self.delta_html_post = [] self.delta_html_pre = [] + # setup boto3 configurations + self.s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + + - def clean_html(self): - pre_scan_html = self.test.pre_scan.html.splitlines() - post_scan_html = self.test.post_scan.html.splitlines() + def clean_html(self) -> None: + # cleans both pre_ and post_ html + # and prepares them for comparison + # retrieveing data from remote s3 + pre_scan_html_raw = requests.get(self.test.pre_scan.html).text + post_scan_html_raw = requests.get(self.test.post_scan.html).text + pre_scan_html = pre_scan_html_raw.splitlines() + post_scan_html = post_scan_html_raw.splitlines() + + # setting watch lists white_list = ['csrfmiddlewaretoken', '',] tags = [ - '', '', '')) + # clean post_scan_html for line in post_scan_html: for item in white_list: if item in line: @@ -59,15 +102,21 @@ def clean_html(self): if sub not in tags: self.post_scan_html.append((sub+'>')) - return + return None + + - def clean_logs(self): + def clean_logs(self) -> None: + # cleans both pre_ and post_ logs + # and prepares them for comparison + + # setting defaults pre_scan_logs_json = self.test.pre_scan.logs post_scan_logs_json = self.test.post_scan.logs order = ("level", "source", "message") - + # cleaning pre_scan_logs for log in pre_scan_logs_json: new_log = {} for label in order: @@ -76,7 +125,7 @@ def clean_logs(self): new_log[label] = log.get(key) self.pre_scan_logs.append(json.dumps(new_log)) - + # cleaning post_scan_logs for log in post_scan_logs_json: new_log = {} for label in order: @@ -85,71 +134,100 @@ def clean_logs(self): new_log[label] = log.get(key) self.post_scan_logs.append(json.dumps(new_log)) - return + return None + + + + def compare_html(self) -> float: + # calculates the similarity of pre and post html + # using SequenceMatcher() - def compare_html(self): + # clean html first self.clean_html() - pre_scan = self.pre_scan_html - post_scan = self.post_scan_html + + # calculate score html_raw_score = SequenceMatcher( - None, pre_scan, post_scan + None, self.pre_scan_html, self.post_scan_html ).ratio() - return html_raw_score + # return score + return html_raw_score if html_raw_score >= 0 else 0 + - def compare_logs(self): + def compare_logs(self) -> float: + # calculates the similarity of pre and post logs + # using SequenceMatcher() + + # clean logs first self.clean_logs() - pre_scan = list(self.pre_scan_logs) - post_scan = list(self.post_scan_logs) + + # calculate score logs_raw_score = SequenceMatcher( - None, pre_scan, post_scan + None, self.pre_scan_logs, self.post_scan_logs ).ratio() - return logs_raw_score + # return score + return logs_raw_score if logs_raw_score >= 0 else 0 - def delta_html(self): + + + def delta_html(self) -> dict: + # Calculates the macro difference in pre_ & post_ html + # (i.e. difference in html nodes
). + # also generates the micro differences + # using self.post_proc_html() + + # calculate macro difference num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html) num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html) if num_html_ratio > 1: num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html) + # building data for post_proc_html() for line in self.post_scan_html: if line not in self.pre_scan_html: self.delta_html_post.append(line) - for line in self.pre_scan_html: if line not in self.post_scan_html: self.delta_html_pre.append(line) - + # get pre_mciro_delta in html pre_micro_delta = self.post_proc_html( self.delta_html_pre, self.delta_html_post ) + # get post_mciro_delta in html post_micro_delta = self.post_proc_html( self.delta_html_post, self.delta_html_pre ) - + # formatting data data = { "num_html_delta": num_html_delta, "delta_html_post": self.delta_html_post, "delta_html_pre": self.delta_html_pre, - "num_html_ratio": num_html_ratio, + "num_html_ratio": num_html_ratio if num_html_ratio >= 0 else 0, "pre_micro_delta": pre_micro_delta, "post_micro_delta": post_micro_delta, } + # return updated data return data - def post_proc_html(self, primary_list, secondary_list): + + + def post_proc_html(self, primary_list: list, secondary_list: list) -> dict: + # generates a list of 8 char long chunks that are in + # the primary_list but not in the secondary_list + + # setting defaults delta_parsed = [] delta_parsed_diff = [] secondary_str = ''.join(str(i) for i in secondary_list) @@ -165,94 +243,122 @@ def post_proc_html(self, primary_list, secondary_list): if block != None and block != '' and block not in secondary_str: delta_parsed_diff.append(block) + # formatting data data = { "delta_parsed": delta_parsed, "delta_parsed_diff": delta_parsed_diff, } + # returning updated data return data - def html_micro_diff_score(self, post_delta_parsed_diff): + def html_micro_diff_score(self, post_delta_parsed_diff: list) -> float: + # Calculates a score by comparing + # post_delta_parsed_diff & pre_delta_parsed_diff + # building pre_delta_parsed_diff pre_delta_parsed_diff = [] for line in self.pre_scan_html: subStrings = re.findall('.{1,8}', line) for sub in subStrings: pre_delta_parsed_diff.append(sub) - + + # calculate score diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff) diff_score = diff_length / len(pre_delta_parsed_diff) - return diff_score + # return score + return diff_score if diff_score >= 0 else 0 - def post_proc_logs(self, log): + def post_proc_logs(self, log: str) -> dict: + # cleaning logs for comparions + # and convert to a dict + + # clean message log = json.loads(log) - log["message"].replace("\"", "\'") - letters = string.digits - timestamp = ''.join(random.choice(letters) for i in range(13)) + log["message"].replace("\"", "\'") + + # generate random timestamp + nums = string.digits + timestamp = ''.join(random.choice(nums) for i in range(13)) log['timestamp'] = timestamp + # return cleaned log return log - def delta_logs(self): + def delta_logs(self) -> dict: + # Calculates scores for log differences + # and builds lists to show diffferences + + # defaults + num_logs_ratio = 1 + delta_logs_post = [] + delta_logs_pre = [] + + # calc nums_log_delta (were there more in post_scan?) num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs) + # calculate ratio if len(self.post_scan_logs) > 0: num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs) if num_logs_ratio > 1: - num_logs_ratio = 1 - else: - num_logs_ratio = 1 + num_logs_ratio = 1 - delta_logs_post = [] + # build list of not present post_scan_logs for log in self.post_scan_logs: if log not in self.pre_scan_logs: log = self.post_proc_logs(log) delta_logs_post.append(log) - - - delta_logs_pre = [] + + # build list of not present pre_scan_logs for log in self.pre_scan_logs: if log not in self.post_scan_logs: log = self.post_proc_logs(log) delta_logs_pre.append(log) + # formatting data data = { "num_logs_delta": num_logs_delta, "delta_logs_post": delta_logs_post, "delta_logs_pre": delta_logs_pre, - "num_logs_ratio": num_logs_ratio, + "num_logs_ratio": num_logs_ratio if num_logs_ratio >= 0 else 0 } + # returning data return data + def delta_lighthouse(self) -> dict: + # calculate the differences in LH + # scores between pre_ and post_ scans - def delta_lighthouse(self): try: + # get pre scores pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo']) pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility']) pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance']) pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices']) - pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) + # pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0 + # get post scores post_seo = int(self.test.post_scan.lighthouse["scores"]['seo']) post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility']) post_performance = int(self.test.post_scan.lighthouse["scores"]['performance']) post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices']) - post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) + # post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0 + # try to get pre and post crux scores try: pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux']) post_crux = int(self.test.post_scan.lighthouse["scores"]['crux']) @@ -262,34 +368,34 @@ def delta_lighthouse(self): post_crux = None crux_delta = 0 + # calculate individual deltas seo_delta = post_seo - pre_seo accessibility_delta = post_accessibility - pre_accessibility performance_delta = post_performance - pre_performance best_practices_delta = post_best_practices - pre_best_practices - pwa_delta = post_pwa - pre_pwa + # pwa_delta = post_pwa - pre_pwa + # calculate averages if post_crux is None: current_average = ( post_seo + post_accessibility + post_best_practices + - post_performance + post_pwa - )/5 - + post_performance # + post_pwa + )/4 old_average = ( pre_seo + pre_accessibility + pre_best_practices + - pre_performance + pre_pwa - )/5 - + pre_performance # + pre_pwa + )/4 else: current_average = ( post_seo + post_accessibility + post_best_practices + - post_performance + post_pwa + post_crux - )/6 - + post_performance + post_crux # + post_pwa + )/5 old_average = ( pre_seo + pre_accessibility + pre_best_practices + - pre_performance + pre_pwa + pre_crux - )/6 + pre_performance + pre_crux # + pre_pwa + )/5 + # calculate difference in averages average_delta = current_average - old_average except: @@ -297,36 +403,40 @@ def delta_lighthouse(self): accessibility_delta = None performance_delta = None best_practices_delta = None - pwa_delta = None + # pwa_delta = None crux_delta = None current_average = None average_delta = None + # formatting data data = { "scores": { "seo_delta": seo_delta, "accessibility_delta": accessibility_delta, "performance_delta": performance_delta, "best_practices_delta": best_practices_delta, - "pwa_delta": pwa_delta, + # "pwa_delta": pwa_delta, "crux_delta": crux_delta, "current_average": current_average, "average_delta": average_delta, } } + # returning data return data + def delta_yellowlab(self) -> dict: + # calculate the differences in YL + # scores between pre_ and post_ scans - - def delta_yellowlab(self): try: + # get pre scores pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore']) pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight']) - pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests']) + pre_images = int(self.test.pre_scan.yellowlab["scores"]['images']) pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity']) pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity']) pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript']) @@ -335,10 +445,11 @@ def delta_yellowlab(self): pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS']) pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts']) pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig']) - + + # get post scores post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore']) post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight']) - post_requests = int(self.test.post_scan.yellowlab["scores"]['requests']) + post_images = int(self.test.post_scan.yellowlab["scores"]['images']) post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity']) post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity']) post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript']) @@ -348,8 +459,9 @@ def delta_yellowlab(self): post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts']) post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig']) + # calculate individual deltas pageWeight_delta = post_pageWeight - pre_pageWeight - requests_delta = post_requests - pre_requests + images_delta = post_images - pre_images domComplexity_delta = post_domComplexity - pre_domComplexity javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity badJavascript_delta = post_badJavascript - pre_badJavascript @@ -359,12 +471,13 @@ def delta_yellowlab(self): fonts_delta = post_fonts - pre_fonts serverConfig_delta = post_serverConfig - pre_serverConfig - average_delta = post_globalScore - pre_globalScore + # get current averag and calc average_delta current_average = post_globalScore - + average_delta = post_globalScore - pre_globalScore + except: pageWeight_delta = None - requests_delta = None + images_delta = None domComplexity_delta = None javascriptComplexity_delta = None badJavascript_delta = None @@ -376,10 +489,11 @@ def delta_yellowlab(self): average_delta = None current_average = None, + # formatting response data = { "scores": { "pageWeight_delta": pageWeight_delta, - "requests_delta": requests_delta, + "images_delta": images_delta, "domComplexity_delta": domComplexity_delta, "javascriptComplexity_delta": javascriptComplexity_delta, "badJavascript_delta": badJavascript_delta, @@ -393,27 +507,223 @@ def delta_yellowlab(self): } } + # returning data return data - def update_site_info(self, test): + + def get_lh_audits_deltas(self, scores: dict) -> str: + # finds and records the changes in LH audit data + # then saves as .json file in s3 and returns + + # defaults + audits = { + "seo":[], + "accessibility": [], + "performance": [], + "pwa": [], + "best_practices": [], + "crux": [] + } + + # get pre & post audits + pre_scan_audits = requests.get(self.test.pre_scan.lighthouse['audits']).json() + post_scan_audits = requests.get(self.test.post_scan.lighthouse['audits']).json() + + # deciding which categories to + # compare based on score + cats = [] + for key in scores: + # checking for a delta score + if '_delta' in key and 'average' not in key: + # check if delta not Zero + if scores[key] is not None: + if float(scores[key]) != 0: + cats.append(str(key).split('_delta')[0]) + + # compare each audit in each of the + # selected categories + for cat in cats: + for audit in post_scan_audits[cat]: + found = False + for aud in pre_scan_audits[cat]: + if audit == aud: + found = True + break + + # record post_ audit if not + # found in pre_ + if not found: + audits[cat].append(audit) + + # save data at .json in s3 + lh_audit_file_uri = self.save_data_to_s3(_data=audits) + + # return uri + return lh_audit_file_uri + + + + + def get_yl_audits_deltas(self, scores: dict) -> str: + # finds and records the changes in YL audit data + # then saves as .json file in s3 and returns + + # defaults + audits = { + "pageWeight":[], + "images": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # get pre & post audits + pre_scan_audits = requests.get(self.test.pre_scan.yellowlab['audits']).json() + post_scan_audits = requests.get(self.test.post_scan.yellowlab['audits']).json() + + # deciding which categories to + # compare based on score + cats = [] + for key in scores: + # checking for a delta score + if '_delta' in key and 'average' not in key: + # check if delta not Zero + if scores[key] is not None: + if float(scores[key]) != 0: + cats.append(str(key).split('_delta')[0]) + + # compare each audit in each of the + # selected categories + for cat in cats: + for audit in post_scan_audits[cat]: + found = False + for aud in pre_scan_audits[cat]: + if audit == aud: + found = True + break + + # record post_ audit if not + # found in pre_ + if not found: + audits[cat].append(audit) + + # save data at .json in s3 + yl_audit_file_uri = self.save_data_to_s3(_data=audits) + + # return uri + return yl_audit_file_uri + + + + + def update_site_info(self, test: object) -> object: + # updates associated Site with + # new Test data + + # get associated site site = test.site - site.info['latest_test']['id'] = str(test.id) - site.info['latest_test']['time_created'] = str(test.time_created) - site.info['latest_test']['time_completed'] = str(test.time_completed) - site.info['latest_test']['score'] = (round(test.score * 100) / 100) - site.save() + # get pages + pages = Page.objects.filter(site=site) + + # get latest tests of pages + tests = [] + for page in pages: + if Test.objects.filter(page=page).exists(): + _test = Test.objects.filter(page=page).exclude( + time_completed=None + ).order_by('-time_completed') + if len(_test) > 0: + if _test[0].score is not None: + tests.append(_test[0].score) + + if len(tests) > 0: + + # calc site average of latest + site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100 + print(f'updating site with new test score -> {site_avg_test_score}') + + # update site info + site.info['latest_test']['id'] = str(test.id) + site.info['latest_test']['time_created'] = str(test.time_created) + site.info['latest_test']['time_completed'] = str(test.time_completed) + site.info['latest_test']['score'] = site_avg_test_score + site.info['latest_test']['status'] = test.status + site.save() + + # returning updated site return site + + + + + def update_page_info(self, test: object) -> object: + # updates associated Page with + # new Test data + + # get page + page = test.page + + # update page info + page.info['latest_test']['id'] = str(test.id) + page.info['latest_test']['time_created'] = str(test.time_created) + page.info['latest_test']['time_completed'] = str(test.time_completed) + page.info['latest_test']['score'] = (round(test.score * 100) / 100) + page.info['latest_test']['status'] = test.status + page.save() + + # return updated page + return page + + + + + def save_data_to_s3(self, _data: dict) -> str: + # Saves passed data as an s3 object and + # returns the remote uri as a str + + # save _data s3 json file + file_id = uuid.uuid4() + with open(f'{file_id}.json', 'w') as fp: + json.dump(_data, fp) + # upload to s3 and return url + data_file = os.path.join(settings.BASE_DIR, f'{file_id}.json') + remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json' + root_path = settings.AWS_S3_URL_PATH + data_file_uri = f"{root_path}/{remote_path}" + + # upload to s3 + with open(data_file, 'rb') as data: + self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + + # remove local copy + os.remove(data_file) + # return uri + return data_file_uri + def run_test(self) -> object: + """ + Runs all the test components specified in the `Test` + and returns the updated `Test` - def run_test(self, index=None): + Expects: None + + Returns: `Test` object + """ # update test obj with scan configs self.test.pre_scan_configs = self.test.pre_scan.configs @@ -442,116 +752,149 @@ def run_test(self, index=None): # default data html_delta_context = None + html_delta_uri = None logs_delta_context = None lighthouse_data = None yellowlab_data = None images_data = None - - + # testing html if 'html' in self.test.type or 'full' in self.test.type: - # scores - html_score = self.compare_html() - delta_html_data = self.delta_html() - num_html_ratio = delta_html_data['num_html_ratio'] - micro_diff_score = self.html_micro_diff_score( + try: + # scores + html_score = self.compare_html() + delta_html_data = self.delta_html() + num_html_ratio = delta_html_data['num_html_ratio'] + micro_diff_score = self.html_micro_diff_score( delta_html_data['post_micro_delta']['delta_parsed_diff'] ) - - # weights - html_score_w = 1 - num_html_w = 1 - micro_diff_w = 2 - - # data - html_delta_context = { - "pre_html_delta": delta_html_data['delta_html_pre'], - "post_html_delta": delta_html_data['delta_html_post'], - "pre_micro_delta": delta_html_data['pre_micro_delta'], - "post_micro_delta": delta_html_data['post_micro_delta'], - } - + + # weights + html_score_w = 1 + num_html_w = 1 + micro_diff_w = 2 + + # data + html_delta_context = { + "pre_html_delta": delta_html_data['delta_html_pre'], + "post_html_delta": delta_html_data['delta_html_post'], + "pre_micro_delta": delta_html_data['pre_micro_delta'], + "post_micro_delta": delta_html_data['post_micro_delta'], + } + + # save and get s3 object uri + html_delta_uri = self.save_data_to_s3(_data=html_delta_context) + print(f'html_delta => {html_delta_uri}') + except Exception as e: + print(e) + micro_diff_w = 0 + num_html_w = 0 + micro_diff_w = 0 + + - + # testing logs if 'logs' in self.test.type or 'full' in self.test.type: - # scores - logs_score = self.compare_logs() - delta_logs_data = self.delta_logs() - num_logs_ratio = delta_logs_data['num_logs_ratio'] - - # weights - logs_score_w = .5 - num_logs_w = 2 - - # data - logs_delta_context = { - "pre_logs_delta": delta_logs_data['delta_logs_pre'], - "post_logs_delta": delta_logs_data['delta_logs_post'], - } - - + try: + # scores + logs_score = self.compare_logs() + delta_logs_data = self.delta_logs() + num_logs_ratio = delta_logs_data['num_logs_ratio'] + + # weights + logs_score_w = .5 + num_logs_w = 2 + + # combined score + combined_logs_score = ((logs_score*logs_score_w) + (num_logs_ratio*num_logs_w))/2.5 + + # data + logs_delta_context = { + "pre_logs_delta": delta_logs_data['delta_logs_pre'], + "post_logs_delta": delta_logs_data['delta_logs_post'], + "combined_logs_score": combined_logs_score + } + except Exception as e: + logs_score_w = 0 + print(e) + + # testing LH if 'lighthouse' in self.test.type or 'full' in self.test.type: - # scores & data - lighthouse_data = self.delta_lighthouse() - lighthouse_avg = lighthouse_data['scores']['average_delta'] - if lighthouse_avg != None and lighthouse_avg > -100: - lighthouse_score = (100 + lighthouse_avg)/100 - if lighthouse_avg != None and lighthouse_avg <= -100: - lighthouse_score = 0 - - # weights - if lighthouse_score == None: + try: + # scores & data + lighthouse_data = self.delta_lighthouse() + lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores']) + lighthouse_data['audits'] = lh_audits_uri + lighthouse_avg = lighthouse_data['scores']['average_delta'] + if lighthouse_avg != None and lighthouse_avg > -100: + lighthouse_score = (100 + lighthouse_avg)/100 + if lighthouse_avg != None and lighthouse_avg <= -100: + lighthouse_score = 0 + + # weights + if lighthouse_score == None: + delta_lh_w = 0 + elif lighthouse_score > 1: + delta_lh_w = 1 + lighthouse_score = 1 + else: + delta_lh_w = 1 + except Exception as e: delta_lh_w = 0 - elif lighthouse_score > 1: - delta_lh_w = 1 - lighthouse_score = 1 - else: - delta_lh_w = 1 - - - + print(e) + + # testing YL if 'yellowlab' in self.test.type or 'full' in self.test.type: - # scores & data - yellowlab_data = self.delta_yellowlab() - yellowlab_avg = yellowlab_data['scores']['average_delta'] - if yellowlab_avg != None and yellowlab_avg > -100: - yellowlab_score = (100 + yellowlab_avg)/100 - if yellowlab_avg != None and yellowlab_avg <= -100: - yellowlab_score = 0 - - # weights - if yellowlab_score == None: + try: + # scores & data + yellowlab_data = self.delta_yellowlab() + yl_audits_uri = self.get_yl_audits_deltas(scores=yellowlab_data['scores']) + yellowlab_data['audits'] = yl_audits_uri + yellowlab_avg = yellowlab_data['scores']['average_delta'] + if yellowlab_avg != None and yellowlab_avg > -100: + yellowlab_score = (100 + yellowlab_avg)/100 + if yellowlab_avg != None and yellowlab_avg <= -100: + yellowlab_score = 0 + + # weights + if yellowlab_score == None: + delta_yl_w = 0 + elif yellowlab_score > 1: + delta_yl_w = 1 + yellowlab_score = 1 + else: + delta_yl_w = 1 + except Exception as e: delta_yl_w = 0 - elif yellowlab_score > 1: - delta_yl_w = 1 - yellowlab_score = 1 - else: - delta_yl_w = 1 - - - + print(e) + + # testing images if 'vrt' in self.test.type or 'full' in self.test.type: - # scores & data - images_data = Image().test(test=self.test, index=index) - if images_data['average_score'] != None: - images_score = images_data['average_score'] / 100 - - # weights - images_w = 4 + try: + # scores & data + images_data = Imager(test=self.test).test_vrt() + if images_data['average_score'] != None: + images_score = images_data['average_score'] / 100 + + # weights + images_w = 4 + except Exception as e: + images_w = 0 + print(e) - + # calculating total weight total_w = ( html_score_w + logs_score_w + num_html_w + num_logs_w + delta_lh_w + micro_diff_w + images_w + delta_yl_w ) - - score = (( + # calculating final weighted average score + score = round(((( (html_score * html_score_w) + (logs_score * logs_score_w) + (num_logs_ratio * num_logs_w) + @@ -560,9 +903,8 @@ def run_test(self, index=None): (yellowlab_score * delta_yl_w) + (micro_diff_score * micro_diff_w) + (images_score * images_w) - ) / total_w) * 100 + ) / total_w) * 100), 2) - print( "Formula was --> ((" + str(html_score*html_score_w) + " + " + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " @@ -571,27 +913,33 @@ def run_test(self, index=None): " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score) ) - + # updating test data self.test.time_completed = datetime.now() - self.test.html_delta = html_delta_context + self.test.html_delta = html_delta_uri self.test.logs_delta = logs_delta_context self.test.lighthouse_delta = lighthouse_data self.test.yellowlab_delta = yellowlab_data self.test.images_delta = images_data self.test.score = score - self.test.component_scores['html'] = (micro_diff_score * 100) - self.test.component_scores['logs'] = (num_logs_ratio * 100) - self.test.component_scores['lighthouse'] = (lighthouse_score * 100) - self.test.component_scores['yellowlab'] = (yellowlab_score * 100) - self.test.component_scores['vrt'] = (images_score * 100) - - + self.test.status = 'passed' if score >= self.test.threshold else 'failed' + self.test.component_scores['html'] = (micro_diff_score * 100) if micro_diff_w != 0 else None + self.test.component_scores['logs'] = (logs_delta_context['combined_logs_score'] * 100) if num_logs_w != 0 else None + self.test.component_scores['lighthouse'] = (lighthouse_score * 100) if delta_lh_w != 0 else None + self.test.component_scores['yellowlab'] = (yellowlab_score * 100) if delta_yl_w != 0 else None + self.test.component_scores['vrt'] = (images_score * 100) if images_w != 0 else None self.test.save() + # updating associated page and site + self.update_page_info(self.test) self.update_site_info(self.test) - return self.test + # create issue if failed + if self.test.status == 'failed' and self.test.post_scan_configs.get('create_issue'): + print('generating new Issue...') + Issuer(test=self.test).build_issue() + # returning updated test + return self.test diff --git a/app/api/utils/updater.py b/app/api/utils/updater.py new file mode 100644 index 00000000..7fe2daeb --- /dev/null +++ b/app/api/utils/updater.py @@ -0,0 +1,224 @@ +from ..models import * +from django.utils import timezone +from django.db import transaction +from cursion import settings + + + + + + +def update_flowrun(**kwargs) -> object: + """ + Updates the `FlowRun`, matching the 'flowrun_id', + with the **kwargs data + + Args: + 'flowrun_id' : str + 'node_index' : int or str, + 'messsage' : str, + 'node_status' : str, + 'objects' : list of dicts + + Returns: + `FlowRun` obj + """ + + # get passed kwargs + flowrun_id = kwargs.get('flowrun_id') + node_index = kwargs.get('node_index') + node_status = kwargs.get('node_status') + message = kwargs.get('message') + objects = kwargs.get('objects') + + with transaction.atomic(): + # lock the row so concurrent workers cannot read-modify-write + # stale copies of nodes/edges/logs. + flowrun = FlowRun.objects.select_for_update().get(id=flowrun_id) + + # Ignore stale worker updates after completion; late async tasks should + # not mutate finished runs. + if flowrun.time_completed is not None: + return flowrun + + # set timestamp + timestamp = timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f') + + # find flowrun.edge by target + def get_edge_by_target(target: str=None) -> dict: + # defaults + edge = None + index = 0 + # find target + for e in flowrun.edges: + if e['target'] == target: + edge = e + break + index+=1 + # return data + return { + 'index': index, + 'edge': edge + } + + # object helpers + def _clean_str(value): + return str(value) if value is not None else None + + def _normalize_object(obj): + normalized = dict(obj or {}) + normalized['parent'] = _clean_str(normalized.get('parent')) + normalized['id'] = _clean_str(normalized.get('id')) + normalized['source_id'] = _clean_str( + normalized.get('source_id', normalized.get('id')) + ) + normalized['track_id'] = _clean_str(normalized.get('track_id')) + + # Transitional defaults while older callers still send legacy shape. + if normalized['track_id'] is None: + if normalized['id'] is not None: + normalized['track_id'] = normalized['id'] + elif normalized['source_id'] is not None: + normalized['track_id'] = f'source:{normalized["source_id"]}' + elif normalized['parent'] is not None: + normalized['track_id'] = f'legacy:{normalized["parent"]}' + + return normalized + + def _same_object(a, b): + # Preferred identity key. + if a.get('track_id') and b.get('track_id'): + return a['track_id'] == b['track_id'] + # Transitional fallback for legacy payloads. + if a.get('id') and b.get('id'): + return a['id'] == b['id'] + # Final legacy fallback. + return a.get('parent') == b.get('parent') + + # update object_list + def add_or_update_objects(object_list, objects): + updated = [_normalize_object(o) for o in (object_list or [])] + for raw_obj in (objects or []): + incoming = _normalize_object(raw_obj) + exists = False + i = 0 + for existing in updated: + if _same_object(existing, incoming): + merged = dict(existing) + merged.update(incoming) + + # Preserve resolved ids when incoming payload is still pending. + if incoming.get('id') is None and existing.get('id') is not None: + merged['id'] = existing.get('id') + # Keep the resolved source identity when incoming payload + # is only a placeholder update. + if existing.get('source_id') is not None: + merged['source_id'] = existing.get('source_id') + if incoming.get('source_id') is None and existing.get('source_id') is not None: + merged['source_id'] = existing.get('source_id') + if incoming.get('track_id') is None and existing.get('track_id') is not None: + merged['track_id'] = existing.get('track_id') + + updated[i] = merged + exists = True + break + i += 1 + + if not exists: + updated.append(incoming) + + return updated + + # check if all objects are complete + def objects_are_complete(object_list): + if len(object_list) == 0: + return True + for obj in object_list: + if obj['status'] == 'working': + return False + return True + + # get collective status of + def get_step_status(object_list): + statuses = [obj['status'] for obj in object_list] + if len(object_list) == 0: + return 'passed' + if 'working' in statuses: + return 'working' + if 'failed' in statuses and 'working' not in statuses: + return 'failed' + return 'passed' + + # update flowrun logs, nodes, & edges + nodes = flowrun.nodes + edges = flowrun.edges + logs = flowrun.logs + + if node_index is not None: + # get node object_list + object_list = nodes[int(node_index)]['data'].get('objects', []) + + # update object_list if objects + if objects: + object_list = add_or_update_objects(object_list, objects) + nodes[int(node_index)]['data']['objects'] = object_list + + # if node_status is provided + if node_status: + nodes[int(node_index)]['data']['status'] = node_status + if node_status != 'working': + nodes[int(node_index)]['data']['time_completed'] = timestamp + + # decide on node status if 'node_status' not provided + if not node_status: + complete = objects_are_complete(object_list) + nodes[int(node_index)]['data']['status'] = get_step_status(object_list) if complete else 'working' + nodes[int(node_index)]['data']['time_completed'] = timestamp if complete else None + + # update current edge if not at flowrun start + if int(node_index) != 0: + edge_index = get_edge_by_target(target=nodes[int(node_index)]['id'])['index'] + edges[edge_index]['animated'] = True if nodes[int(node_index)]['data']['status'] == 'working' else False + edges[edge_index]['style'] = {'stroke': "#60a5fa"} if nodes[int(node_index)]['data']['status'] == 'working' else None + + # added messages to logs + if message: + + # loop through multiple messages if passed: + for msg in message.split(','): + + # check for empty string + if msg and len(msg) > 0: + + # update current logs + logs.append({ + 'timestamp': timestamp, + 'message': msg, + 'step': nodes[int(node_index)]['id'] if node_index else logs[-1]['step'] + }) + + # sort new logs + logs = sorted(logs, key=lambda l: (int(l['step']))) + + # save updates while holding the row lock + flowrun.nodes = nodes + flowrun.edges = edges + flowrun.logs = logs + flowrun.save() + + # run_next() should execute for updater-driven changes. + # keep this explicit so progression does not rely solely on signal timing. + if settings.LOCATION == 'us': + flowrun_id_str = str(flowrun.id) + + def _run_next(): + try: + from .flowr import Flowr + Flowr(flowrun_id=flowrun_id_str).run_next() + except Exception as e: + print(f'[update_flowrun] run_next trigger error: {e}') + + transaction.on_commit(_run_next) + + # return updated flowrun + return flowrun diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py index eb119c4f..af11358a 100644 --- a/app/api/utils/verify.py +++ b/app/api/utils/verify.py @@ -1,30 +1,39 @@ -import os, requests, json +import os, requests, json, signal + + + + + def verify(): - username = os.environ.get('ADMIN_USER') - email = os.environ.get('ADMIN_EMAIL') - password = os.environ.get('ADMIN_PASS') - cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792' - url = 'https://scanerr.io/verify' - - - headers = { - "Content-Type": "application/json", - "Authorization" : cred - } - data = { - "username": username, - "email": email, - "password": password, - } - - res = requests.get( - url=url, - headers=headers, - params=data - ).json() - - if res['verified']: - return - else: - os.abort() \ No newline at end of file + + if os.environ.get('MODE') == 'selfhost': + username = os.environ.get('ADMIN_USER') + email = os.environ.get('ADMIN_EMAIL') + license_key = os.environ.get('LICENSE_KEY') + api_root = os.environ.get('API_URL_ROOT') + client_root = os.environ.get('CLIENT_URL_ROOT') + url = f'{os.environ.get('LANDING_URL_ROOT')}/ops/verify' + + headers = { + "Content-Type": "application/json", + } + + data = { + "username": username, + "email": email, + "license_key": license_key, + "api_root": api_root, + "client_root": client_root + } + + res = requests.get( + url=url, + headers=headers, + params=data + ).json() + + if res.get('verified'): + return + else: + os.kill(os.getpid(), signal.SIGTERM) \ No newline at end of file diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py index f94ea864..3534f1e7 100644 --- a/app/api/utils/wordpress.py +++ b/app/api/utils/wordpress.py @@ -1,4 +1,4 @@ -from .driver_s import driver_init, driver_wait +from .driver import driver_init, driver_wait from selenium import webdriver from selenium.webdriver.support.ui import Select from selenium.webdriver.common.keys import Keys @@ -11,7 +11,6 @@ - class Wordpress(): @@ -61,7 +60,7 @@ def login(self): ''' Tries to log into a WP site with given credentials. - returns --> True / False + Returns: True / False ''' @@ -378,7 +377,7 @@ def launch_migration(self): ''' Launches the migration plugin once Activated. - returns --> True / False + Returns: True / False ''' @@ -422,7 +421,7 @@ def run_migration(self): and begins updating the associated `Process` with data from the page. - returns --> True / False + Returns: True / False ''' @@ -518,7 +517,7 @@ def run_migration(self): #

Your migration is complete!

# get full page div if new_progress >= 100 or done_text in self.driver.page_source: - self.process.successful = True + self.process.success = True self.process.time_completed = datetime.now() done = True diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py deleted file mode 100644 index 889a00ff..00000000 --- a/app/api/utils/wordpress_p.py +++ /dev/null @@ -1,571 +0,0 @@ -from .driver_p import driver_init -import time, asyncio, uuid -from ..models import * -from datetime import datetime -from asgiref.sync import sync_to_async - - - - - - - -class Wordpress(): - - - def __init__( - self, - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - wait_time, - process_id - ): - # set all global vars - self.login_url = login_url - self.username = username - self.password = password - self.email_address = email_address - self.destination_url = destination_url - self.sftp_address = sftp_address - self.dbname = dbname - self.sftp_username = sftp_username - self.sftp_password = sftp_password - self.process = Process.objects.get(id=process_id) - self.native_lang = 'en' - - if not admin_url.endswith('/'): - admin_url = admin_url + '/' - self.admin_url = admin_url - - if wait_time is None: - self.wait_time = 30 - else: - self.wait_time = wait_time - - self.navWaitOpt = { - 'timeout': self.wait_time * 1000, - 'waitUntil': 'domcontentloaded' - } - - - async def login(self): - - ''' - Tries to log into a WP site with given credentials. - - returns --> True / False - - ''' - - print('begining login method for ' + self.login_url) - - - self.driver = await driver_init(wait_time=self.wait_time) - - # init page obj - self.page = await self.driver.newPage() - page_options = { - 'waitUntil': 'networkidle0', - 'timeout': self.wait_time * 1000 - } - - try: - await self.page.goto(self.login_url, page_options) - try: - await self.page.xpath('//*[@id="user_login"]') - print('found login form') - except: - try: - jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]') - await jetpack[0].click() - await self.page.xpath('//*[@id="user_login"]') - print('found login form') - except: - try: - login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]") - await login_link[0].click() - await self.page.xpath('//*[@id="user_login"]') - print('found login form') - except: - print('unable to locate login form at this path') - await self.driver.close() - return False - - - except: - print('unable to locate login form at this path') - await self.driver.close() - return False - - user_name_elem = await self.page.xpath('//*[@id="user_login"]') - await user_name_elem[0].click(clickCount=3) - await self.page.keyboard.type(self.username) - time.sleep(1) - passworword_elem = await self.page.xpath('//*[@id="user_pass"]') - await passworword_elem[0].click(clickCount=3) - await self.page.keyboard.type(self.password) - time.sleep(1) - await self.page.keyboard.press('Enter') - await self.page.waitForNavigation(self.navWaitOpt) - - - try: - try: - verify_email = await self.page.xpath('//*[@id="correct-admin-email"]') - print('need to verify email') - await verify_email[0].click() - print('clicked verify') - except: - pass - - print('done with login attempt') - - try: - await self.page.xpath('//*[@id="login_error"]') - print('found login error') - await self.page.reload() - - print('trying login again') - user_name_elem = await self.page.xpath('//*[@id="user_login"]') - await user_name_elem[0].click(clickCount=3) - await self.page.keyboard.type(self.username) - time.sleep(1) - passworword_elem = await self.page.xpath('//*[@id="user_pass"]') - await passworword_elem[0].click(clickCount=3) - await self.page.keyboard.type(self.password) - time.sleep(1) - await self.page.keyboard.press('Enter') - await self.page.waitForNavigation(self.navWaitOpt) - - - try: - await self.page.xpath('//*[@id="login_error"]') - print('found login error again') - print('counld not login to this site') - except: - print('no login errors') - - except: - print('no login errors') - - except: - print('counld not login to this site') - - await self.driver.close() - return False - - - # removing alerts - try: - deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]') - await deny_btn[0].click() - print('removed alert') - except: - pass - try: - # checking if url location is wp-admin - admin_link = '/wp-admin/' - current_url = self.page.url - print('current url -> ' + current_url) - if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url: - print('inside wp-admin') - else: - print('not in wp-admin - navigating there now') - admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]') - admin_link = await admin_btn[0].querySelector('a') - await admin_link[0].click(clickCount=2) - print('clicked dashboard link') - await self.page.waitForNavigation(self.navWaitOpt) - - - except: - print('could not login') - await self.driver.close() - return False - - - return True - - - - - - async def begin_lang_check(self): - - try: - # navigate to settings - s_url = 'options-general.php' - try: - settings_menu = await self.page.xpath('//*[@id="menu-settings"]') - await settings_menu[0].click() - print('clicked settings menu') - await self.page.waitForNavigation(self.navWaitOpt) - settings = await self.page.xpath('.//a[@href="'+s_url+'"]') - await settings[0].click() - print('clicked settings tab') - await self.page.waitForNavigation(self.navWaitOpt) - - - except: - await self.page.goto(self.page.url + s_url) - await self.page.waitForNavigation(self.navWaitOpt) - - # finding and recording current native language - lang_selector = await self.page.xpath('//*[@id="WPLANG"]') - optgroup = await lang_selector[0].querySelector('optgroup') - selected_lang = await optgroup.xpath('.//option[@selected="selected"]') - default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue() - default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue() - print("defalut lang value is " + str(default_lang)) - - if default_lang != 'en': - - # selecting english - await lang_selector[0].select('en_CA') - print('selected english') - - # saving settings - save_btn = await self.page.xpath('//*[@id="submit"]') - await save_btn[0].click() - print('saved lang to english') - - self.native_lang = default_lang_value - return True - - else: - self.native_lang = 'en' - - - except: - print('error in changing language') - return False - - - - - - - async def end_lang_check(self): - - if self.native_lang != 'en': - - try: - # navigate to settings - s_url = 'options-general.php' - try: - settings_menu = await self.page.xpath('//*[@id="menu-settings"]') - await settings_menu[0].click() - print('clicked settings menu') - await self.page.waitForNavigation(self.navWaitOpt) - settings = await self.page.xpath('.//a[@href="'+s_url+'"]') - await settings[0].click() - print('clicked settings tab') - await self.page.waitForNavigation(self.navWaitOpt) - - except: - await self.page.goto(self.page.url + s_url) - await self.page.waitForNavigation(self.navWaitOpt) - - # selecting native lang - lang_selector = await self.page.xpath('//*[@id="WPLANG"]') - await lang_selector[0].select(self.native_lang) - print('selected native_lang') - - # saving settings - save_btn = await self.page.xpath('//*[@id="submit"]') - await save_btn[0].click() - print('saved native lang') - - except: - await self.driver.close() - return False - - await self.driver.close() - return True - - - - async def install_plugin(self, plugin_name): - - # setting url for link naving - plugin_menu_page = 'plugins.php' - add_plugin_page = 'plugin-install.php' - - # navigating to plugin page - try: - print('trying click method') - plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]') - await plugin_menu[0].click() - await self.page.waitForNavigation(self.navWaitOpt) - p_url = 'plugins.php' - plugins = await self.page.xpath('.//a[@href="'+p_url+'"]') - await plugins[0].click() - print('clicked plugin menu') - await self.page.waitForNavigation(self.navWaitOpt) - - - # looking for dependencies in plugin table - time.sleep(10) - form = await self.page.xpath('//*[@id="bulk-action-form"]') - pluginTable = await form[0].querySelector('tbody') - tableText = await (await pluginTable.getProperty('textContent')).jsonValue() - - except: - print('trying link method for navigation') - try: - await self.page.goto(self.admin_link + plugin_menu_page) - await self.page.waitForNavigation(self.navWaitOpt) - - time.sleep(10) - # looking for dependencies in plugin table - form = await self.page.xpath('//*[@id="bulk-action-form"]') - pluginTable = await form[0].querySelector('tbody') - tableText = await (await pluginTable.getProperty('textContent')).jsonValue() - except: - print('unable to find plugin table') - await self.driver.close() - return False - - if plugin_name not in tableText: - try: - print('plugin not present, preparing to install') - - time.sleep(2) - print('navigating to add plugins page') - - try: - url = 'plugin-install.php' - add_plugin = await self.page.xpath('//a[@href="'+url+'"]') - await add_plugin[0].click(clickCount=2) - print('clicked add plugin link') - await self.page.waitForNavigation(self.navWaitOpt) - - time.sleep(5) - except: - await self.page.goto(self.admin_url + add_plugin_page) - await self.page.waitForNavigation(self.navWaitOpt) - - time.sleep(5) - - - # searching for plugin - search_form = await self.page.xpath('//input[@type="search"]') - await search_form[0].click(clickCount=3) - await self.page.keyboard.type(plugin_name) - time.sleep(1) - await self.page.keyboard.press('Enter') - time.sleep(3) - - ##### Clicking "install" plugin ###### - install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly - await install[0].click(clickCount=2) - print('clicked -install plugin-') - time.sleep(30) - - - #### Clicking "activate" plugin ###### - await self.page.reload() - print('reloading page') - try: - await self.page.waitForNavigation(self.navWaitOpt) - except: - pass - activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly - await activate[0].click(clickCount=2) - print('clicked -Activate plugin-') - time.sleep(30) - print('Dependencies installed sucessfully') - return True - - except: - print('failed dependency installation') - await self.driver.close() - return False - - else: - print('plugin already installed') - return True - - - @sync_to_async - def update_process(self, successful=False, info_url=None, time_completed=None, progress=None): - if info_url is not None: - self.process.info_url = info_url - self.process.successful = successful - if time_completed is not None: - self.process.time_completed = time_completed - if progress is not None: - self.process.progress = progress - - self.process.save() - return - - - - async def launch_migration(self): - ''' - Launches the migration plugin once Activated. - - returns --> True / False - - ''' - - # setting url for link naving - migrate_page = 'admin.php?page=cloudways' - current_url = self.page.url - - if not current_url.endswith("cloudways"): - print('navigating to migration page') - if self.admin_url.endswith('/'): - await self.page.goto(f'{self.admin_url}{migrate_page}') - else: - await self.page.goto(f'{self.admin_url}/{migrate_page}') - time.sleep(10) - - - # wait for cloudways email field to become visible - # entering self.email_address in field - email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input') - await email[0].click(clickCount=3) - await self.page.keyboard.type(self.email_address) - print('entered cloudways email') - - # checking T&S checbox - checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]') - await checkbox[0].click(clickCount=1) - print('checked T&S agreement') - - # clicking submit to launch migration plugin - m_button = await self.page.xpath('//*[@id="migratesubmit"]') - await m_button[0].click(clickCount=1) - print('clicked migrate button') - - return True - - - async def run_migration(self): - ''' - Enters data on migration page, initiates miration - and begins updating the associated `Process` with data - from the page. - - returns --> True / False - - ''' - - # check for page to fully load - print('waiting 10 sec for new page to load') - time.sleep(10) - ## enter all necessary data in each field - await self.page.waitForNavigation(self.navWaitOpt) - - # get_element_by_name="address" -> self.destination_url - destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]') - await destination_url[0].click(clickCount=3) - await self.page.keyboard.type(self.destination_url) - print(f'dest_url as -> {self.destination_url}') - time.sleep(2) - - # get_element_by_name="newurl" -> self.sftp_address - sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]') - await sftp_address[0].click(clickCount=3) - await self.page.keyboard.type(self.sftp_address) - print(f'sftp_address as -> {self.sftp_address}') - time.sleep(2) - - # get_element_by_name="appfolder" -> self.dbname - dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]') - await dbname[0].click(clickCount=3) - await self.page.keyboard.type(self.dbname) - print(f'dbname as -> {self.dbname}') - time.sleep(2) - - # get_element_by_name="username" -> self.sftp_username - sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]') - await sftp_username[0].click(clickCount=3) - await self.page.keyboard.type(self.sftp_username) - print(f'sftp_username as -> {self.sftp_username}') - time.sleep(2) - - # get_element_by_name="passwd" -> self.sftp_password - sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]') - await sftp_password[0].click(clickCount=3) - await self.page.keyboard.type(self.sftp_password) - print(f'sftp_password as -> {self.sftp_password}') - time.sleep(2) - - print('entered all creds') - - # submit data - await self.page.keyboard.press('Enter') - print('pressed enter key') - - - - # update self.process with info_url - info_url = self.page.url - await self.update_process(info_url=info_url) - - - done = False - done_text = 'Your migration is complete!' - new_progress = 0 - print(f'current url -> {self.page.url}') - while not done: - - # checking for progres bar - try: - raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]') - new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue() - new_progress = float(new_progress.split('%')[0]) - except Exception as e: - # print(e) - pass - - - # update self.process - await self.update_process(progress=new_progress) - - # check if new_progress is 100% - page_content = await self.page.content() - if new_progress >= 100 or done_text in page_content: - time_completed = datetime.now() - await self.update_process(successful=True, time_completed=time_completed, progress=100) - done = True - - # checking for process errors - if 'alert alert-danger' in page_content: - done = True - time_completed = datetime.now() - await self.update_process(time_completed=time_completed) - print('found an error - ending process') - return False - - time.sleep(1) - - - return True - - - - - - - async def run_full(self, plugin_name): - data = await self.login() - data = await self.begin_lang_check() - data = await self.install_plugin(plugin_name) - # data = await self.end_lang_check() - data = await self.launch_migration() - data = await self.run_migration() - await self.driver.close() - return data - diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py index ea3e11b6..8480bb57 100644 --- a/app/api/utils/yellowlab.py +++ b/app/api/utils/yellowlab.py @@ -1,141 +1,281 @@ -import subprocess, json +import subprocess, json, uuid, boto3, os, requests, time from ..models import Site, Scan +from .devices import get_device +from cursion import settings + + + class Yellowlab(): - """Initializes Yellow Lab Tools CLI and runs an audit of the site""" + """ + Initializes Yellow Lab Tools CLI and runs an audit of the site + + Use self.get_data() to init a run + """ - def __init__(self, site=None, configs=None): - self.site = site - self.configs = configs + def __init__(self, scan=None): + self.scan = scan + self.site = self.scan.site + self.page = self.scan.page + self.configs = scan.configs + self.audits_url = '' + self.device_type = get_device( + scan.configs['browser'], + scan.configs['device'] + )['type'] + + # initial audits object + self.audits = { + "pageWeight": [], + "images": [], + "domComplexity": [], + "javascriptComplexity": [], + "badJavascript": [], + "jQuery": [], + "cssComplexity": [], + "badCSS": [], + "fonts": [], + "serverConfig": [], + } + + # initial scores object + self.scores = { + "globalScore": None, + "pageWeight": None, + "images": None, + "domComplexity": None, + "javascriptComplexity": None, + "badJavascript": None, + "jQuery": None, + "cssComplexity": None, + "badCSS": None, + "fonts": None, + "serverConfig": None, + } - def init_audit(self): + def yellowlab_cli(self): + """ + Serves as the CLI method for collecting YL metrics. + Creates a sub process running yellowlabtools CLI + + Returns: raw YL data (Dict) + """ + + print(f'starting YL with device type -> {self.device_type}') + + # initiating subprocess for YLT CLI proc = subprocess.Popen([ - 'yellowlabtools', - self.site.site_url, - f'--device={self.configs["device"]}' + 'yellowlabtools', + self.page.page_url, + f'--device={self.device_type}' ], stdout=subprocess.PIPE, user='app', ) + + # retrieving data from process stdout_value = proc.communicate()[0] - return stdout_value + + # converting stdout str into Dict + stdout_json = json.loads(stdout_value) + return stdout_json + + + + def yellowlab_api(self) -> dict: + """ + Serves as the API method for collecting YL metrics. + Sends API requests to http://yellowlab:8383 + or localhost:8383 + + Returns: raw YL data (Dict) + """ + + headers = { + "Content-Type": "application/json", + "Connection": "close" + } + payload = { + "url": self.page.page_url, + "waitForResponse": True, + "device": self.device_type + } + + def curl_post(url, data): + cmd = [ + "curl", "-s", "-X", "POST", + "-H", f"Content-Type: {headers['Content-Type']}", + "-H", f"Connection: {headers['Connection']}", + "--data", json.dumps(data), + url + ] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + return json.loads(result.stdout) + + def curl_get(url): + cmd = [ + "curl", "-s", "-X", "GET", + "-H", f"Connection: {headers['Connection']}", + url + ] + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + return json.loads(result.stdout) + + # setting up initial request + root = settings.YELLOWLAB_ROOT + res = curl_post(f"{root}/api/runs", payload) + + # retrieve runId & pod_ip if present + run_id = res["runId"] + pod_ip = res.get("pod_ip") + new_root = f"http://{pod_ip}:8383" if pod_ip else root + + wait_time = 0 + max_wait = 1200 + done = False + + # waiting for run to complete + while not done and wait_time < max_wait: + + # sending run request check + status_res = curl_get(f"{new_root}/api/runs/{run_id}") + status = status_res["run"]["status"]["statusCode"] + position = status_res["run"]["status"].get("position") + + # checking status + if status == "awaiting" and position: + max_wait = max(max_wait, 120 * position) + elif status == "complete": + done = True + elif status == "failed": + raise RuntimeError("YellowLab run failed") + + # incrementing time + time.sleep(5) + wait_time += 5 + + # Step 3: Retrieve results + result = curl_get(f"{new_root}/api/results/{run_id}") + return result + + + + + def process_data(self, stdout_json: dict) -> dict: + """ + Accepts JSON data from either CLI or API method + and parses into usable Cursion data. + + Expects the following: + stdout_json: or json from output + + Returns: formatted YL data + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # iterating through categories to get relevant yl_audits + # and store them in their respective `audits = {}` obj + for cat in self.audits: + cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] + for a in cat_audits: + try: + audit = stdout_json["rules"][a] + self.audits[cat].append(audit) + except: + pass + + # get scores from each category + for key in self.scores: + if key == 'globalScore': + score_value = stdout_json["scoreProfiles"]["generic"]["globalScore"] + if score_value is None: + continue + self.scores['globalScore'] = score_value + else: + score_value = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"] + if score_value is None: + continue + self.scores[key] = score_value + + + # save audits data as json file + file_id = uuid.uuid4() + with open(f'{file_id}.json', 'w') as fp: + json.dump(self.audits, fp) + + # upload to s3 and return url + audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json') + remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json' + root_path = settings.AWS_S3_URL_PATH + audits_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(audit_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"} + ) + # remove local copy + os.remove(audit_file) + + # updating opjects + self.audits_url = audits_url + + data = { + "scores": self.scores, + "audits": self.audits_url, + "failed": False + } + + # returning data + return data def get_data(self): - try: - stdout_value = self.init_audit() - # decode bytes into string - stdout_string = stdout_value.decode('iso-8859-1') + + scan_complete = False + failed = True + attempts = 0 - if len(stdout_string) != 0: - if 'Runtime error encountered' in stdout_string: - error = {'error': 'yellowlab ran into a problem',} - return error - - stdout_json = json.loads(stdout_value) - - # initial audits object - audits = { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - } - - # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj - for cat in audits: - cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"] - for a in cat_audits: - try: - audit = stdout_json["rules"][a] - audits[cat].append(audit) - except: - pass - - - # get scores from each category - globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"] - pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"] - requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"] - domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"] - javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"] - badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"] - jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"] - cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"] - badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"] - fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"] - serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"] + # trying yellowlab scan until success or 2 attempts + while not scan_complete and attempts < 2: + + try: + # CLI on first attempt + if attempts < 1: + raw_data = self.yellowlab_cli() + self.process_data(stdout_json=raw_data) - scores = { - "globalScore": globalScore, - "pageWeight": pageWeight_score, - "requests": requests_score, - "domComplexity": domComplexity_score, - "javascriptComplexity": javascriptComplexity_score, - "badJavascript": badJavascript_score, - "jQuery": jQuery_score, - "cssComplexity": cssComplexity_score, - "badCSS": badCSS_score, - "fonts": fonts_score, - "serverConfig": serverConfig_score, - } - - data = { - "scores": scores, - "audits": audits, - "failed": False - } + # API after first attempt + if attempts >= 1: + raw_data = self.yellowlab_api() + self.process_data(stdout_json=raw_data) - else: - raise RuntimeError - - except Exception as e: - print(e) - - scores = { - "globalScore": None, - "pageWeight": None, - "requests": None, - "domComplexity": None, - "javascriptComplexity": None, - "badJavascript": None, - "jQuery": None, - "cssComplexity": None, - "badCSS": None, - "fonts": None, - "serverConfig": None, - } - - audits = { - "pageWeight": [], - "requests": [], - "domComplexity": [], - "javascriptComplexity": [], - "badJavascript": [], - "jQuery": [], - "cssComplexity": [], - "badCSS": [], - "fonts": [], - "serverConfig": [], - } - - data = { - "scores": scores, - "audits": audits, - "failed": True - } + scan_complete = True + failed = False + + except Exception as e: + print(f'YELLOWLAB FAILED (attempt {attempts}) --> {e}') + scan_complete = False + failed = True + attempts += 1 + + data = { + "scores": self.scores, + "audits": self.audits_url if self.audits_url != '' else None, + "failed": failed + } + # returning final data return data - - diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py deleted file mode 100644 index 10392b78..00000000 --- a/app/api/v1/auth/alerts.py +++ /dev/null @@ -1,166 +0,0 @@ -from django.core.mail import send_mail, send_mass_mail -from django.contrib.auth.models import User -from django.template.loader import render_to_string -from datetime import date -import os, operator -from ...models import * -from django.utils.html import strip_tags -from django.contrib.auth.models import User -from rest_framework_simplejwt.tokens import RefreshToken -from rest_framework.response import Response -from ...utils.alerts import sendgrid_email -from scanerr import settings - - - - -def send_reset_link(email): - if User.objects.filter(email=email).exists(): - user = User.objects.get(email=email) - token = RefreshToken.for_user(user) - access_token = str(token.access_token) - reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token) - subject = 'Rest Password' - title = 'Reset Password' - pre_header = 'Reset Password' - pre_content = 'Click the link below to reset your password.' - - subject = subject - context = { - 'title' : title, - 'subject' : subject, - 'email': email, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : reset_link, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'Rest my password', - 'content' : '', - 'signature' : '- Cheers!', - } - - sendgrid_email(message_obj=context) - - # html_message = render_to_string('api/alert_with_button.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [email], - # html_message = html_message, - # fail_silently = True, - # ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - - - -def send_invite_link(member): - if Member.objects.filter(email=member.email, status="pending").exists(): - member = Member.objects.get(email=member.email) - link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}' - subject = 'Scanerr Invite' - title = 'Scanerr Invite' - pre_header = 'Scanerr Invite' - pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!' - - subject = subject - context = { - 'title' : title, - 'subject' : subject, - 'email': member.email, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : link, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'button_text' : 'Accept Invite', - 'content' : '', - 'signature' : '- Cheers!', - } - - sendgrid_email(message_obj=context) - - # html_message = render_to_string('api/alert_with_button.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [member.email], - # html_message = html_message, - # fail_silently = True, - # ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data - - - - - -def send_remove_alert(member): - if Member.objects.filter(email=member.email, status="removed").exists(): - member = Member.objects.get(email=member.email) - subject = 'Removed From Account' - title = 'Removed From Account' - pre_header = 'Removed From Account' - pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.' - - subject = subject - context = { - 'title' : title, - 'subject' : subject, - 'email': member.email, - 'pre_header' : pre_header, - 'pre_content' : pre_content, - 'object_url' : None, - 'home_page' : os.environ.get('CLIENT_URL_ROOT'), - 'content' : '', - 'signature' : '- Cheers!', - } - - sendgrid_email(message_obj=context) - - # html_message = render_to_string('api/alert_no_button.html', context) - # plain_message = strip_tags(html_message) - # send_mail( - # from_email = os.getenv('EMAIL_HOST_USER'), - # subject = subject, - # message = plain_message, - # recipient_list = [member.email], - # html_message = html_message, - # fail_silently = True, - # ) - - data = { - 'success': True - } - - else: - data = { - 'success': False - } - - return data \ No newline at end of file diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py index 77f4d9ab..fa7f0a5a 100644 --- a/app/api/v1/auth/serializers.py +++ b/app/api/v1/auth/serializers.py @@ -10,6 +10,11 @@ from rest_framework import routers, serializers, viewsets from rest_framework.fields import UUIDField + + + + + kwargs = { 'allow_null': False, 'read_only': True, @@ -18,10 +23,13 @@ + class UserSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = User - fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login'] + fields = ['id', 'username', 'email', 'password', 'is_active', + 'date_joined', 'last_login', 'first_name', 'last_name'] + @@ -44,13 +52,16 @@ def validate(self, attrs): return data + + class RegisterSerializer(UserSerializer): password = serializers.CharField(max_length=128, min_length=8, write_only=True, required=True) email = serializers.EmailField(required=True, write_only=True, max_length=128) class Meta: model = User - fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login'] + fields = ['id', 'username', 'email', 'password', 'is_active', + 'date_joined', 'last_login', 'first_name', 'last_name'] def create(self, validated_data): try: @@ -61,19 +72,22 @@ def create(self, validated_data): + class AccountSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) class Meta: model = Account - fields = ['id', 'active', 'time_created', 'type', + fields = ['id', 'active', 'time_created', 'type', 'cust_id', 'sub_id', 'product_id', 'price_id', 'slack', - 'user', 'code', 'name', + 'user', 'code', 'name', 'price_amount', + 'configs', 'meta', 'usage', 'info', 'license_key', ] + class MemberSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) @@ -81,9 +95,10 @@ class MemberSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Member - fields = ['id', 'status', 'time_created', 'type', - 'email', 'type', 'user', 'account', + fields = ['id', 'status', 'time_created', 'type', 'phone', + 'email', 'type', 'user', 'account', 'permissions', ] - \ No newline at end of file + + diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py index 96fa4eec..8447367c 100644 --- a/app/api/v1/auth/services.py +++ b/app/api/v1/auth/services.py @@ -1,26 +1,30 @@ -import requests, os, subprocess, secrets -from typing import Dict, Any -from scanerr import settings -from django.http import HttpResponse -from django.db import transaction -from rest_framework import status, serializers -from rest_framework_simplejwt.tokens import RefreshToken + + from django.core.exceptions import ValidationError -from django.forms.models import model_to_dict from django.contrib.auth.models import User +from django.contrib.auth.password_validation import validate_password from django.shortcuts import get_object_or_404 +from django.utils import timezone +from django.db.models import Q from rest_framework.authtoken.models import Token -from ...models import Account, Card, Member -from ..ops.services import record_api_call +from rest_framework.response import Response +from rest_framework.pagination import LimitOffsetPagination +from rest_framework import status +from rest_framework_simplejwt.tokens import RefreshToken from slack_sdk.oauth import AuthorizeUrlGenerator -from slack_sdk.oauth.installation_store import FileInstallationStore, Installation +from slack_sdk.oauth.installation_store import FileInstallationStore from slack_sdk.oauth.state_store import FileOAuthStateStore from slack_sdk.web import WebClient +from ...models import Account, Member, Site, get_permissions_default +from ..ops.services import record_api_call from .serializers import * -from .alerts import * -from rest_framework.response import Response -from rest_framework.pagination import LimitOffsetPagination -from django.contrib.auth.middleware import get_user +from ...utils.alerts import send_reset_link +from ...tasks import send_invite_link_bg, send_remove_alert_bg, create_prospect +from cursion import settings +import requests, os, secrets, signal + + + @@ -29,108 +33,366 @@ GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo' -def jwt_login(*, user: User): - refresh = RefreshToken.for_user(user) - access = str(refresh.access_token) - refresh = str(refresh) + + +### ------ Begin User Services ------ ### + + + + +def register_user(request: object) -> object: + """ + Creates a User object and returns a request + + Expects the following: + 'email' : str, + 'password' : str, + 'first_name' : str, + 'last_name' : str, + + Returns: + 'user' : dict, + 'token' : str, + 'refresh' : str, + 'api_token' : str + """ + + # get data + password = request.data.get('password') + username = request.data.get('username') + first_name = request.data.get('first_name') + last_name = request.data.get('last_name') + + # validate requests + if (password is None or len(password) == 0) or \ + (username is None or len(username) == 0): + data = {'detail': 'Must provide an email and password.'} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) + + if User.objects.filter(username=username).exists(): + data = {'detail': 'Account already exists.'} + return Response(data=data, status=status.HTTP_409_CONFLICT) - if Token.objects.filter(user=user).exists(): - api_token = Token.objects.get(user=user) - else: - api_token = Token.objects.create(user=user) + # validate password and create user + try: + # check password + if validate_password(password) == None: + + # create user + user = User.objects.create( + username=username, + email=username, + first_name=first_name, + last_name=last_name, + last_login=timezone.now() + ) + + # setting password + user.set_password(raw_password=password) + user.save() + + # generating JWTs + refresh = RefreshToken.for_user(user) + + # generate API token + api_token = Token.objects.create(user=user) + + # returning data + data = { + 'user': UserSerializer(user).data, + 'token': str(refresh.access_token), + 'refresh': str(refresh), + 'api_token': str(api_token.key) + } + return Response(data=data, status=status.HTTP_201_CREATED) + + except: + data = {'detail': 'Please choose a stronger password.'} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) + + + + +def login_user(request: object) -> object: + """ + Authenticates a User object and returns a request + + Expects the following: + 'username' : str, (same as email unless 'admin') + 'password' : str + + Returns: + 'user' : dict, + 'token' : str, + 'refresh' : str, + 'api_token' : str + """ - if user.is_active == True: - is_active = 'true' + # get data + password = request.data.get('password') + email = request.data.get('email') + + # validate requests + if (password is None or len(password) == 0) or \ + (email is None or len(email) == 0): + data = {'detail': 'Must provide an email and password.'} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) + + # setting defalt response + data = {'detail': 'No account found with the given credentials.'} + + # checking is User exists via provided email / username + if User.objects.filter(Q(username=email) | Q(email=email)): + + # retrieving User obj + if User.objects.filter(username=email): + user = User.objects.get(username=email) + else: + user = User.objects.get(email=email) + + # validating password + if user.check_password(password): + + # generating JWTs + refresh = RefreshToken.for_user(user) + + # get API token + api_token = Token.objects.get(user=user) + + # update user last_login + user.last_login = timezone.now() + + # returning data + data = { + 'user': UserSerializer(user).data, + 'token': str(refresh.access_token), + 'refresh': str(refresh), + 'api_token': str(api_token.key) + } + return Response(data=data, status=status.HTTP_201_CREATED) + else: + return Response(data=data, status=status.HTTP_401_UNAUTHORIZED) else: - is_acive = 'false' + return Response(data=data, status=status.HTTP_401_UNAUTHORIZED) - param_string = str( - '?access='+access+'&refresh='+refresh+ - '&username='+user.username+'&id='+str(user.id)+ - '&email='+user.email+'&is_active='+is_active+ - '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+ - '&api_token='+str(api_token.key) - ) - lead_string = str(settings.CLIENT_URL_ROOT+'/google-confirm') + + +def update_user(request: object) -> object: + """ + Updates the User with the passed "email". + + Args: + 'request': object + } + + Returns: + HTTP Response object + """ + + # get request data + email = request.data.get('email') + user = request.user + member = Member.objects.get(user=user) + + # check if an email is already associated with a user + if User.objects.filter(email=email).exists() and user.email != email: + return Response(status=status.HTTP_417_EXPECTATION_FAILED) - redirect_url = lead_string + param_string + # update user email + if user.username != 'admin': + user.username = email + user.email = email + user.save() - return redirect_url + # update member email + member.email = email + member.save() + # serialize and return + data = UserSerializer(user).data + return Response(data, status=status.HTTP_200_OK) -def user_create(email, password=None, **extra_fields) -> User: - extra_fields = { - 'is_staff': False, - 'is_superuser': False, - **extra_fields - } - user = User.objects.create( - username=email, - email=email, - **extra_fields - ) +def update_password(request: object) -> object: + """ + Updates the User with the passed "password". - # creating API token - Token.objects.create(user=user) + Args: + 'request': object + } - user.set_unusable_password() - user.full_clean() - user.save() + Returns: + HTTP Response object + """ - return user + # get request data + password = request.data.get('password') + user = request.user + try: + # validate password + if validate_password(password, user=user) == None: + # udpdate password + user.set_password(password) + user.save() -def create_user_token(request): - # creating New API token - if Token.objects.filter(user=request.user).exists(): - old_token = Token.objects.get(user=request.user) - old_token.delete() + # return success + return Response(status=status.HTTP_200_OK) - api_token = Token.objects.create(user=request.user) - data = {'api_token': api_token.key,} - return Response(data, status=status.HTTP_200_OK) + except: + # respond with error + return Response(status=status.HTTP_417_EXPECTATION_FAILED) -def user_get_or_create(*, email: str, **extra_data): - user = User.objects.filter(email=email).first() +def send_reset_email(request: object) -> object: + """ + Sends a password reset email to the + User that matches the passed "email". - if user: - return user + Args: + 'request': object + + Returns: + HTTP Response object + """ - return user_create(email=email, **extra_data) + # get request data + email = request.data.get('email') + # send + resp = send_reset_link(email) + + if resp.get('success') == True: + return Response(status=status.HTTP_200_OK) + + return Response(status=status.HTTP_404_NOT_FOUND) -def google_validate_id_token(*, id_token: str): - # Reference: https://developers.google.com/identity/sign-in/web/backend-auth#verify-the-integrity-of-the-id-token - response = requests.get( - GOOGLE_ID_TOKEN_INFO_URL, - params={'id_token': id_token} + +### ------ Begin GoogleAuth Services ------ ### + + + + +def jwt_login(*, user: object) -> str: + """ + Gets JWTs for passed "user" and builds a + redirect url for returning user params back to + Cursion.client + + Expect: { + 'user': object + + Returns: str + """ + + # get JWTs for user + refresh = RefreshToken.for_user(user) + token = str(refresh.access_token) + refresh = str(refresh) + + # create API token if none exists + if not Token.objects.filter(user=user).exists(): + Token.objects.create(user=user) + + # get API token + api_token = Token.objects.get(user=user) + + # setting user active + is_active = str(user.is_active).lower() + + # update user last_login + user.last_login = timezone.now() + user.save() + + # building params for redirect + param_string = str( + '?token='+str(token)+'&refresh='+str(refresh)+ + '&username='+str(user.username)+'&id='+str(user.id)+ + '&email='+str(user.email)+'&is_active='+str(is_active)+ + '&created='+str(user.date_joined)+'&updated='+str(timezone.now())+ + '&api_token='+str(api_token.key) ) - if not response.ok: - raise ValidationError('id_token is invalid.') + # build redirect url + redirect_url = f'{settings.CLIENT_URL_ROOT}/google-confirm{param_string}' + + # return redirect + return redirect_url - audience = response.json()['aud'] - if audience != settings.GOOGLE_OAUTH2_CLIENT_ID: - raise ValidationError('Invalid audience.') - return True + +def get_or_create_user(email: str, **extra_fields) -> object: + """ + Creates a new `User` with the passed "email". + + Args: + 'email' : str, + + Returns: User object + """ + + # trying to find user + user = User.objects.filter(email=email).first() + + # return user if found + if user: + return user + + # formating extra passed data + extras = { + 'is_staff': False, + 'is_superuser': False, + } + + # format user's names + if extra_fields.get('first_name') is not None: + extras['first_name'] = extra_fields.get('first_name') + if extra_fields.get('last_name') is not None: + extras['last_name'] = extra_fields.get('last_name') + + # create the user + user = User.objects.create( + username=email, + email=email, + last_login=timezone.now(), + **extra_fields + ) + + # creating API token + Token.objects.create(user=user) + + # setting password + user.set_unusable_password() + user.full_clean() + user.save() + + # returning new User + return user def google_get_access_token(*, code: str, redirect_uri: str) -> str: - # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#obtainingaccesstokens + """ + Get an access token from Google OAuth2 API + + Args: + 'code' : str, + 'redirect_uri' : str + + Returns: str + """ + + # format request data data = { 'code': code, 'client_id': settings.GOOGLE_OAUTH2_CLIENT_ID, @@ -139,39 +401,124 @@ def google_get_access_token(*, code: str, redirect_uri: str) -> str: 'grant_type': 'authorization_code' } + # send google request response = requests.post(GOOGLE_ACCESS_TOKEN_OBTAIN_URL, data=data) if not response.ok: raise ValidationError('Failed to obtain access token from Google.') + # parse access_token access_token = response.json()['access_token'] + # return access token return access_token -def google_get_user_info(*, access_token: str) -> Dict[str, Any]: - # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#callinganapi +def google_get_user_info(*, access_token: str) -> dict: + """ + Gets User info from google OAuth2 API + + Args: + 'access_token' + + Returns: dict + """ + + # send request response = requests.get( GOOGLE_USER_INFO_URL, params={'access_token': access_token} ) + # check for errors if not response.ok: raise ValidationError('Failed to obtain user info from Google.') + # return user info return response.json() -def slack_oauth_middleware(request, user): - code = request.GET['code'] - account = Account.objects.get(user=user) +def google_login(request: object) -> str: + """ + Authenticates and Creates a new User + with Google OAuth + + Args: + 'request': object + + Returns: str + """ + + # get request data + code = request.GET.get('code') + error = request.GET.get('error') + + # build login url + login_url = f'{settings.CLIENT_URL_ROOT}/login' + + # catch error and return + if error or not code: + params = urlencode({'error': error}) + error_url = f'{login_url}?{params}' + return error_url + + # build redirect url + redirect_uri = f'{settings.API_URL_ROOT}/v1/auth/google' + + # get access token + access_token = google_get_access_token(code=code, redirect_uri=redirect_uri) + + # get user data + user_data = google_get_user_info(access_token=access_token) + + # build user profile + profile_data = { + 'email': user_data['email'], + 'first_name': user_data.get('given_name', ''), + 'last_name': user_data.get('family_name', ''), + } + + # get or create user and authenticate + user = get_or_create_user(**profile_data) + confirm_url = jwt_login(user=user) + + # returning confirm url + return confirm_url + + + + +### ------ Begin Slack Services ------ ### + + + + +def slack_oauth_middleware(request: object) -> object: + """ + Used to update `Account` once "account.admin" + has integrated Slack + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + code = request.GET.get('code') + + # get account + account = Account.objects.get(user=request.user) + # init slack webclient client = WebClient() + # send slack client request response = client.oauth_v2_access( client_id=os.environ.get('SLACK_CLIENT_ID'), client_secret=os.environ.get('SLACK_CLIENT_SECRET'), @@ -187,6 +534,7 @@ def slack_oauth_middleware(request, user): account.slack['slack_channel_name'] = response['incoming_webhook']['channel'] account.save() + # serialize and return serializer_context = {'request': request,} serialized = AccountSerializer(account, context=serializer_context) data = serialized.data @@ -196,12 +544,29 @@ def slack_oauth_middleware(request, user): -def slack_oauth_init(request, user): - if Account.objects.filter(user=user).exists(): - account = Account.objects.get(user=user) +def slack_oauth_init(request: object) -> object: + """ + Used to authenticate with Slack + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # check if account exists + if Account.objects.filter(user=request.user).exists(): + + # get account + account = Account.objects.get(user=request.user) + + # check if slackk integrated if not account.slack['slack_channel_name']: + # Issue and consume state parameter value on the server-side. state_store = FileOAuthStateStore(expiration_seconds=300, base_dir="./data") + # Persist installation data and lookup it by IDs. installation_store = FileInstallationStore(base_dir="./data") @@ -213,73 +578,115 @@ def slack_oauth_init(request, user): # Generate a random value and store it on the server-side state = state_store.issue() - # https://slack.com/oauth/v2/authorize?state=(generated value)&client_id={client_id}&scope=app_mentions:read,chat:write&user_scope=search:read url = authorize_url_generator.generate(state) - data = { - 'url': url, - } + + # return data + data = {'url': url} return Response(data, status=status.HTTP_200_OK) + # return error else: - data = { - 'reason': 'slack already integrated', - } + data = {'reason': 'slack integrated'} return Response(data, status=status.HTTP_409_CONFLICT) + # return error else: - data = { - 'reason': 'account not yet setup', - } + data = {'reason': 'account not setup'} return Response(data, status=status.HTTP_404_NOT_FOUND) -def t7e(request): - if request.GET.get('cred') == \ - 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792': - os.abort() - subprocess.Popen(['pkill -f gunicorn'], - stdout=subprocess.PIPE, - user='app', - ) +### ------ Begin Account Services ------ ### -def create_or_update_account(request=None, *args, **kwargs): - # get posted data +def create_or_update_account(request: object=None, *args, **kwargs) -> object: + """ + Creates or Updates an `Account` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data if request is not None: - user = request.user _id = request.data.get('id') name = request.data.get('name') active = request.data.get('active') type = request.data.get('type') code = request.data.get('code') - max_sites = request.data.get('max_sites') cust_id = request.data.get('cust_id') sub_id = request.data.get('sub_id') product_id = request.data.get('product_id') price_id = request.data.get('price_id') + price_amount = request.data.get('price_amount') + interval = request.data.get('interval') + sites_allowed = request.data.get('sites_allowed') + pages_allowed = request.data.get('pages_allowed') + schedules_allowed = request.data.get('schedules_allowed') + retention_days = request.data.get('retention_days') + scans_allowed = request.data.get('scans_allowed') + tests_allowed = request.data.get('tests_allowed') + caseruns_allowed = request.data.get('caseruns_allowed') + flowruns_allowed = request.data.get('flowruns_allowed') + nodes_allowed = request.data.get('nodes_allowed') + conditions_allowed = request.data.get('conditions_allowed') + sites = request.data.get('sites') + schedules = request.data.get('schedules') + scans = request.data.get('scans') + tests = request.data.get('tests') + caseruns = request.data.get('caseruns') + flowruns = request.data.get('flowruns') slack = request.data.get('slack') + configs = request.data.get('configs') + meta = request.data.get('meta') + info = request.data.get('info') + user = request.user + # get kwargs data if request is None: - user = kwargs.get('user') _id = kwargs.get('id') name = kwargs.get('name') active = kwargs.get('active') type = kwargs.get('type') code = kwargs.get('code') - max_sites = kwargs.get('max_sites') cust_id = kwargs.get('cust_id') sub_id = kwargs.get('sub_id') product_id = kwargs.get('product_id') price_id = kwargs.get('price_id') + price_amount = kwargs.get('price_amount') + interval = kwargs.get('interval') + sites_allowed = kwargs.get('sites_allowed') + pages_allowed = kwargs.get('pages_allowed') + schedules_allowed = kwargs.get('schedules_allowed') + retention_days = kwargs.get('retention_days') + scans_allowed = kwargs.get('scans_allowed') + tests_allowed = kwargs.get('tests_allowed') + caseruns_allowed = kwargs.get('caseruns_allowed') + flowruns_allowed = kwargs.get('flowruns_allowed') + nodes_allowed = kwargs.get('nodes_allowed') + conditions_allowed = kwargs.get('conditions_allowed') + sites = kwargs.get('sites') + schedules = kwargs.get('schedules') + scans = kwargs.get('scans') + tests = kwargs.get('tests') + caseruns = kwargs.get('caseruns') + flowruns = kwargs.get('flowruns') slack = kwargs.get('slack') + configs = kwargs.get('configs') + meta = kwargs.get('meta') + info = kwargs.get('info') + user_id = kwargs.get('user') + user = User.objects.get(id=user_id) - + # getting account if id present if _id is not None: - if not Account.objects.filter(id=_id).exists(): + if not Account.objects.filter(id=_id, user=user).exists(): data = {'reason': 'account not found',} record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) @@ -294,8 +701,6 @@ def create_or_update_account(request=None, *args, **kwargs): account.type = type if code is not None: account.code = code - if max_sites is not None: - account.max_sites = max_sites if cust_id is not None: account.cust_id = cust_id if sub_id is not None: @@ -304,33 +709,104 @@ def create_or_update_account(request=None, *args, **kwargs): account.product_id = product_id if price_id is not None: account.price_id = price_id + if price_amount is not None: + account.price_amount = price_amount + if interval is not None: + account.interval = interval + if scans_allowed is not None: + account.usage['scans_allowed'] = scans_allowed + if tests_allowed is not None: + account.usage['tests_allowed'] = tests_allowed + if caseruns_allowed is not None: + account.usage['caseruns_allowed'] = caseruns_allowed + if flowruns_allowed is not None: + account.usage['flowruns_allowed'] = flowruns_allowed + if sites_allowed is not None: + account.usage['sites_allowed'] = sites_allowed + if pages_allowed is not None: + account.usage['pages_allowed'] = pages_allowed + if schedules_allowed is not None: + account.usage['schedules_allowed'] = schedules_allowed + if nodes_allowed is not None: + account.usage['nodes_allowed'] = nodes_allowed + if conditions_allowed is not None: + account.usage['conditions_allowed'] = conditions_allowed + if retention_days is not None: + account.usage['retention_days'] = retention_days + if sites is not None: + account.usage['sites'] = sites + if schedules is not None: + account.usage['schedules'] = schedules + if scans is not None: + account.usage['scans'] = scans + if tests is not None: + account.usage['tests'] = tests + if caseruns is not None: + account.usage['caseruns'] = caseruns + if flowruns is not None: + account.usage['flowruns'] = flowruns if slack is not None: account.slack = slack + if configs is not None: + account.configs = configs + if meta is not None: + account.meta = meta + if info is not None: + account.info = info # saving updated info account.save() - - + # create new account if not exists if _id is None: + # create account code if code is None: code = secrets.token_urlsafe(16) + # create account license_key + license_key = 'cursion-license-' + secrets.token_hex(32) + + # build usage + usage = { + 'sites': 0, + 'schedules': 0, + 'scans': 0, + 'tests': 0, + 'caseruns': 0, + 'flowruns': 0, + 'sites_allowed': sites_allowed if sites_allowed else 1, + 'pages_allowed': pages_allowed if pages_allowed else 3, + 'schedules_allowed': schedules_allowed if schedules_allowed else 1, + 'scans_allowed': scans_allowed if scans_allowed else 30, + 'tests_allowed': tests_allowed if tests_allowed else 30, + 'caseruns_allowed': caseruns_allowed if caseruns_allowed else 15, + 'flowruns_allowed': flowruns_allowed if flowruns_allowed else 5, + 'nodes_allowed': nodes_allowed if nodes_allowed else 4, + 'conditions_allowed': conditions_allowed if conditions_allowed else 1, + 'retention_days': retention_days if retention_days else 15, + } + + # create new account account = Account.objects.create( user=user, name=name, active=True, + license_key=license_key, type=type, code=code, - max_sites=max_sites, cust_id=cust_id, sub_id=sub_id, product_id=product_id, - price_id=price_id + price_id=price_id, + usage=usage, ) + + # create proepsct + queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + create_prospect.apply_async(kwargs={'user_email': str(user.email)}, queue=queue, routing_key=queue) - + # serialize and return serializer_context = {'request': request,} serialized = AccountSerializer(account, context=serializer_context) data = serialized.data @@ -340,51 +816,152 @@ def create_or_update_account(request=None, *args, **kwargs): -def get_account(request=None, id=None, *args, **kwargs): - user = request.user - account_id = request.query_params.get('id') +def get_account(request: object) -> object: + """ + Gets the `Account` associated with the passed user - if id is not None: - account = get_object_or_404(Account, pk=id) - - if account_id is not None: - account = get_object_or_404(Account, pk=account_id) + Args: + 'request': object + + Returns: + HTTP Response object + """ - if account_id is None and id is None: - if not Member.objects.filter(user=user).exists(): - data = {'reason': 'account not found',} - return Response(data, status=status.HTTP_404_NOT_FOUND) - account = Member.objects.get(user=user).account + # get user + user = request.user - if not Member.objects.filter(account=account, user=user).exists(): - data = {'reason': 'you cannot retrieve an Account you are not a member of',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # check `Member` of User + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'account not found'} + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # get member and account + member = Member.objects.get(user=user) + account = member.account + # serialize and return serializer_context = {'request': request,} serialized = AccountSerializer(account, context=serializer_context) data = serialized.data - record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) -def get_account_members(request=None, id=None, *args, **kwargs): - user = request.user - account_id = request.query_params.get('id') - mem_acct = Member.objects.get(user=user).account +def create_user_token(request: object) -> object: + """ + Creates a new API token for the passed "user" - if id is not None: - account = get_object_or_404(Account, pk=id) + Args: + 'request': object + + Returns: + HTTP Response object + """ - if mem_acct != account: - data = {'reason': 'you cannot retrieve an Account you are not a member of',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # delete old token if exists + if Token.objects.filter(user=request.user).exists(): + old_token = Token.objects.get(user=request.user) + old_token.delete() + + # creating New API token + api_token = Token.objects.create(user=request.user) + + # return response + data = {'api_token': api_token.key,} + return Response(data, status=status.HTTP_200_OK) + + + + +def get_account_license(request: object) -> object: + """ + Checks if Account is type "selfhost" and returns + rquested ENV data + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + license_key = request.data.get('license_key') + + # set defaults + success = False + data = {} + + # check key + if Account.objects.filter(license_key=license_key).exists(): + + # build data + data = { + 'GOOGLE_CRUX_KEY' : os.environ.get('GOOGLE_CRUX_KEY'), + 'TWILIO_SID' : os.environ.get('TWILIO_SID'), + 'TWILIO_AUTH_TOKEN' : os.environ.get('TWILIO_AUTH_TOKEN'), + 'SENDGRID_API_KEY' : os.environ.get('SENDGRID_API_KEY'), + 'DEFAULT_TEMPLATE' : os.environ.get('DEFAULT_TEMPLATE'), + 'DEFAULT_TEMPLATE_NO_BUTTON' : os.environ.get('DEFAULT_TEMPLATE_NO_BUTTON'), + 'AUTOMATION_TEMPLATE' : os.environ.get('AUTOMATION_TEMPLATE'), + 'SLACK_APP_ID' : os.environ.get('SLACK_APP_ID'), + 'SLACK_CLIENT_ID' : os.environ.get('SLACK_CLIENT_ID'), + 'SLACK_CLIENT_SECRET' : os.environ.get('SLACK_CLIENT_SECRET'), + 'SLACK_SIGNING_SECRET' : os.environ.get('SLACK_SIGNING_SECRET'), + 'SLACK_VERIFICATION_TOKEN' : os.environ.get('SLACK_VERIFICATION_TOKEN'), + 'SLACK_BOT_TOKEN' : os.environ.get('SLACK_BOT_TOKEN'), + 'AWS_ACCESS_KEY_ID' : os.environ.get('AWS_ACCESS_KEY_ID'), + 'AWS_SECRET_ACCESS_KEY' : os.environ.get('AWS_SECRET_ACCESS_KEY'), + 'GPT_API_KEY' : os.environ.get('GPT_API_KEY') + } + + # update success + success = True + + # return response + data = { + 'success': success, + 'data': data + } + return Response(data, status=status.HTTP_200_OK) + + + + +### ------ Begin Member Services ------ ### + + + + +def get_account_members(request: object, *args, **kwargs) -> object: + """ + Get a list of `Members` associated with the + `Account` of the passed "user" + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get user + user = request.user + + # check `Member` of User + if not Member.objects.filter(user=user).exists(): + data = {'reason': 'account not found'} + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # get member and account + member = Member.objects.get(user=user) + account = member.account + + # get members members = Member.objects.filter(account=account) + # serialize and return paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(members, request) serializer_context = {'request': request,} @@ -395,25 +972,31 @@ def get_account_members(request=None, id=None, *args, **kwargs): -def create_or_update_member(request=None, *args, **kwargs): - # get posted data +def create_or_update_member(request: object=None) -> object: + """ + Creates or Updates a `Member` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data if request is not None: user = request.user _id = request.data.get('id') + send_invite = request.data.get('send_invite') account = request.data.get('account') _status = request.data.get('status') - type = request.data.get('type') + _type = request.data.get('type') email = request.data.get('email') + phone = request.data.get('phone') code = request.data.get('code') + permissions = request.data.get('permissions') - if request is None: - user = kwargs.get('user') - account = kwargs.get('account') - _status = kwargs.get('status') - type = kwargs.get('type') - email = kwargs.get('email') - code = kwargs.get('code') - + # checking account if account is not None: if Account.objects.filter(id=account).exists(): account = Account.objects.get(id=account) @@ -422,6 +1005,7 @@ def create_or_update_member(request=None, *args, **kwargs): record_api_call(request, data, '404') return Response(data, status=status.HTTP_404_NOT_FOUND) + # checking for member if _id is not None: if not Member.objects.filter(id=_id).exists(): data = {'reason': 'member not found',} @@ -434,11 +1018,18 @@ def create_or_update_member(request=None, *args, **kwargs): member.account = account if email is not None: member.email = email + if phone is not None: + member.phone = phone if user is not None and user.username == member.email: member.user = user - if type is not None: - member.type = type + if _type is not None: + member.type = _type + if permissions is not None: + member.permissions = permissions + + # updating status if _status is not None: + # checking if user has valid code for membership if _status == 'active' and code != member.account.code: data = {'reason': 'member not authorized',} @@ -449,24 +1040,36 @@ def create_or_update_member(request=None, *args, **kwargs): # saving updated info member.save() + # create new Member if _id is None: + + # get permissonions or default + _permissions = permissions if permissions else get_permissions_default() + member = Member.objects.create( email=email, + phone=phone, status=_status, - type=type, + type=_type, account=account, + permissions=_permissions ) - if _status == 'pending': - send_invite_link(member) + # sending invite link + if _status == 'pending' and send_invite: + queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + send_invite_link_bg.apply_async(kwargs={'member_id': str(member.id)}, queue=queue, routing_key=queue) + # sending removed alert and deleting if _status == 'removed': - send_remove_alert(member) - member.delete() + # method also deletes member + queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + send_remove_alert_bg.apply_async(kwargs={'member_id': str(member.id)}, queue=queue, routing_key=queue) data = {'message': 'Member removed'} response = Response(data, status=status.HTTP_200_OK) return response + # serialize and return serializer_context = {'request': request,} serialized = MemberSerializer(member, context=serializer_context) data = serialized.data @@ -476,29 +1079,149 @@ def create_or_update_member(request=None, *args, **kwargs): -def get_member(request=None, id=None, *args, **kwargs): +def get_member(request: object=None, id: str=None) -> object: + """ + Get a single member via passed "user" or "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and member_id user = request.user member_id = request.query_params.get('id') + # checking if member exists if id is not None: member = get_object_or_404(Member, pk=id) - if member_id is not None: member = get_object_or_404(Member, pk=member_id) + # getting user's Member object if exists if member_id is None and id is None: if not Member.objects.filter(user=user).exists(): data = {'reason': 'member not found',} return Response(data, status=status.HTTP_404_NOT_FOUND) member = Member.objects.get(user=user) + # checking that member is assoicated with user if member.user != user and member.account.user != user: data = {'reason': 'you cannot retrieve a Member you are not affiliated with',} record_api_call(request, data, '401') return Response(data, status=status.HTTP_403_FORBIDDEN) + # serialize and return serializer_context = {'request': request,} serialized = MemberSerializer(member, context=serializer_context) data = serialized.data record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) \ No newline at end of file + return Response(data, status=status.HTTP_200_OK) + + + + +### ------ Begin Prospect Services ------ ### + + + + +def get_prospects(request: object) -> object: + """ + This pulls all admin Members and + builds a list to reflect the needed + attributes for `Landing.api.Prospect` + + Args: + 'request': object + + Returns: + 'count': int total number of prospects + 'results': list of Prospect objects + """ + + try: + # check if request.user is admin + if request.user.username != 'admin': + return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN) + except: + return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN) + + # get all Accounts + accounts = Account.objects.all().exclude(user__username='admin') + + # iterate throgh accounts + # and build list + results = [] + count = len(accounts) + for account in accounts: + + # determinig user's 'status' + if account.type == 'free': + if Site.objects.filter(account=account).exists(): + _status = 'warm' # account has one site onboarded + else: + _status = 'cold' # account is free but no site onboarded + if account.type != 'free': + if account.active: + _status = 'customer' # account is active and paid + else: + _status = 'warm' # account is paused and paid + if account.type == 'new': + _status = 'cold' # account has not onboarded + if account.type == 'selfhost': + _status = 'customer' + + # get admin member + member = Member.objects.filter(account=account, type='admin')[0] + + # building prospect + prospect = { + 'first_name': account.user.first_name, + 'last_name': account.user.last_name, + 'email': account.user.email, + 'phone': member.phone, + 'status': _status, + 'info': account.info, + 'meta': account.meta, + 'license_key': account.license_key + } + + # adding to results + results.append(prospect) + + # building response + data = { + 'count': count, + 'results': results + } + + # returning response + return Response(data, status=status.HTTP_200_OK) + + + + +def t7e(request: object) -> None: + """ + Helper function for validation & verification + + Args: + 'request': object + + Returns: None + """ + + # validating + if request.query_params.get('license_key') == os.environ.get('LICENSE_KEY'): + + # terminating + try: + os.kill(os.getpid(), signal.SIGTERM) + except Exception as e: + return Response({'success': False}, status=status.HTTP_200_OK) + + diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py index f4a377a9..336e7d06 100644 --- a/app/api/v1/auth/urls.py +++ b/app/api/v1/auth/urls.py @@ -1,22 +1,25 @@ from django.urls import path, include from . import views as views from rest_framework.authtoken.views import obtain_auth_token -from rest_framework import ( - routers, serializers, viewsets, -) +from rest_framework import routers + -router = routers.DefaultRouter() -# auth routes -router.register(r'login', views.LoginViewSet, basename='auth_login') -router.register(r'register', views.RegistrationViewSet, basename='auth_register') +# refresh route +router = routers.DefaultRouter() router.register(r'refresh', views.RefreshViewSet, basename='auth_refresh') + + urlpatterns = [ path('', include(router.urls)), + path('login', views.Login.as_view(), name='login'), + path('register', views.Register.as_view(), name='register'), + path('login/', views.Login.as_view(), name='login'), + path('register/', views.Register.as_view(), name='register'), path('api-auth', include('rest_framework.urls', namespace='rest_framework')), path('api-token-auth', obtain_auth_token, name='api_token_auth'), path('google', views.GoogleLoginApi.as_view(), name='auth_google'), @@ -27,9 +30,9 @@ path('token', views.ApiToken.as_view(), name='token'), path('verify', views.Verify.as_view(), name='verify'), path('account', views.Account.as_view(), name='account'), - path('account/', views.Account.as_view(), name='account-detail'), path('account//members', views.AccountMembers.as_view(), name='account-members'), + path('account/license', views.AccountLicense.as_view(), name='account-license'), path('member', views.Member.as_view(), name='member'), path('member/', views.Member.as_view(), name='member-detail'), - + path('prospect', views.Prospect.as_view(), name='prospect'), ] \ No newline at end of file diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py index 6ea2da15..4441e371 100644 --- a/app/api/v1/auth/views.py +++ b/app/api/v1/auth/views.py @@ -1,120 +1,77 @@ from rest_framework.response import Response -from django.contrib.auth.password_validation import validate_password -from rest_framework_simplejwt.views import TokenObtainPairView from rest_framework_simplejwt.views import TokenRefreshView -from rest_framework.viewsets import ModelViewSet, ViewSet -from rest_framework.permissions import AllowAny +from rest_framework.viewsets import ViewSet +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.views import APIView -from rest_framework import status, serializers -from rest_framework_simplejwt.tokens import RefreshToken, AccessToken -from rest_framework_simplejwt.models import TokenUser -from rest_framework.authtoken.models import Token +from rest_framework import status from rest_framework_simplejwt.exceptions import TokenError, InvalidToken -from .serializers import LoginSerializer, RegisterSerializer, UserSerializer -from scanerr import settings from django.shortcuts import redirect -from django.contrib.auth.models import User -from .alerts import send_reset_link -from ...models import Account, Member -from datetime import timedelta, datetime from .services import * -import os, stripe, json -class LoginViewSet(ModelViewSet, TokenObtainPairView): - serializer_class = LoginSerializer - permission_classes = (AllowAny,) - http_method_names = ['post'] - def create(self, request, *args, **kwargs): - - serializer = self.get_serializer(data=request.data) - try: - serializer.is_valid(raise_exception=True) - except TokenError as e: - raise InvalidToken(e.args[0]) - - return Response(serializer.validated_data, status=status.HTTP_200_OK) - - -class RegistrationViewSet(ModelViewSet, TokenObtainPairView): - serializer_class = RegisterSerializer - permission_classes = (AllowAny,) - http_method_names = ['post'] - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - user = serializer.save() - refresh = RefreshToken.for_user(user) - # creating API token - api_token = Token.objects.create(user=user) - res = { - "refresh": str(refresh), - "access": str(refresh.access_token), - } +### ------ Begin User Views ------ ### - return Response({ - "user": serializer.data, - "refresh": res["refresh"], - "token": res["access"], - "api_token": api_token.key, - }, status=status.HTTP_201_CREATED) -class ApiToken(APIView): +class Login(APIView): permission_classes = (AllowAny,) - http_method_names = ['get'] + http_method_names = ['post',] + authentication_classes = [] - def get(self, request): - response = create_user_token(request) + def post(self, request): + response = login_user(request=request) return response -class Verify(APIView): - authentication_classes = [] + +class Register(APIView): permission_classes = (AllowAny,) - http_method_names = ['get'] + http_method_names = ['post',] + authentication_classes = [] - def get(self, request): - response = t7e(request) + def post(self, request): + response = register_user(request=request) return response + class RefreshViewSet(ViewSet, TokenRefreshView): permission_classes = (AllowAny,) http_method_names = ['post'] def create(self, request, *args, **kwargs): + + # get request data serializer = self.get_serializer(data=request.data) + # validate refresh token and create new access try: serializer.is_valid(raise_exception=True) except TokenError as e: raise InvalidToken(e.args[0]) + # return response return Response(serializer.validated_data, status=status.HTTP_200_OK) + class GetResetLink(APIView): permission_classes = (AllowAny,) http_method_names = ['post',] authentication_classes = [] def post(self, request): - email = request.data['email'] - response = send_reset_link(email) - - if response['success'] == True: - return Response(status=status.HTTP_200_OK) - else: - return Response(status=status.HTTP_404_NOT_FOUND) + response = send_reset_email(request) + return response + @@ -123,82 +80,52 @@ class ResetPassword(APIView): http_method_names = ['post',] def post(self, request): - password = request.data['password'] - user = request.user - try: - if validate_password(password, user=user) == None: - user.set_password(password) - user.save() - return Response(status=status.HTTP_200_OK) - except: - return Response(status=status.HTTP_417_EXPECTATION_FAILED) + response = update_password(request) + return response - class UpdateUser(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): - email = request.data['email'] - user = request.user - try: - if User.objects.filter(email=email).exists(): - return Response(status=status.HTTP_417_EXPECTATION_FAILED) - user.username = email - user.email = email - user.save() - data = UserSerializer(user).data - return Response(data, status=status.HTTP_200_OK) - except: - return Response(status=status.HTTP_417_EXPECTATION_FAILED) + response = update_user(request) + return response -class GoogleLoginApi(APIView): - authentication_classes = [] +class ApiToken(APIView): permission_classes = (AllowAny,) - class InputSerializer(serializers.Serializer): - code = serializers.CharField(required=False) - error = serializers.CharField(required=False) + http_method_names = ['get'] - def get(self, request, *args, **kwargs): - input_serializer = self.InputSerializer(data=request.GET) - input_serializer.is_valid(raise_exception=True) + def get(self, request): + response = create_user_token(request) + return response - validated_data = input_serializer.validated_data - code = validated_data.get('code') - error = validated_data.get('error') - login_url = f'{settings.CLIENT_URL_ROOT}/login' - if error or not code: - params = urlencode({'error': error}) - return redirect(f'{login_url}?{params}') +### ------ Begin GoogleAuth Views ------ ### - domain = settings.API_URL_ROOT - api_uri = '/v1/auth/google' - redirect_uri = f'{domain}{api_uri}' - access_token = google_get_access_token(code=code, redirect_uri=redirect_uri) - user_data = google_get_user_info(access_token=access_token) - profile_data = { - 'email': user_data['email'], - 'first_name': user_data.get('given_name', ''), - 'last_name': user_data.get('family_name', ''), - } +class GoogleLoginApi(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + def get(self, request, *args, **kwargs): + confirm_url = google_login(request) + return redirect(confirm_url) - user = user_get_or_create(**profile_data) - confirm_url = jwt_login(user=user) - return redirect(confirm_url) + + +### ------ Begin Slack Views ------ ### + @@ -207,44 +134,65 @@ class SlackOauth(APIView): http_method_names = ['get', 'post'] def post(self, request, *args, **kwargs): - user = request.user - response = slack_oauth_init(request, user) + response = slack_oauth_init(request) return response def get(self, request, *args, **kwargs): - user = request.user - response = slack_oauth_middleware(request, user) + response = slack_oauth_middleware(request) return response +### ------ Begin Account Views ------ ### + + + class Account(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'post'] - def post(self, request, *args, **kwargs ): + def post(self, request): response = create_or_update_account(request) return response - def get(self, request, id=None, *args, **kwargs): - response = get_account(request, id) + def get(self, request): + response = get_account(request) return response + class AccountMembers(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get',] - def get(self, request, id=None, *args, **kwargs): - response = get_account_members(request, id) + def get(self, request, *args, **kwargs): + response = get_account_members(request) return response -class Member(APIView): + + +class AccountLicense(APIView): + authentication_classes = [] permission_classes = (AllowAny,) + http_method_names = ['post'] + + def post(self, request): + response = get_account_license(request) + return response + + + + +### ------ Begin Member Views ------ ### + + + +class Member(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'post'] def post(self, request, *args, **kwargs ): @@ -255,4 +203,36 @@ def get(self, request, id=None, *args, **kwargs): response = get_member(request, id) return response - + + + +### ------ Begin External Views ------ ### + + + + +class Prospect(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_prospects(request) + return response + + + + +class Verify(APIView): + authentication_classes = [] + permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = t7e(request) + return response + + + + + + diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py new file mode 100644 index 00000000..64331ab5 --- /dev/null +++ b/app/api/v1/billing/services.py @@ -0,0 +1,929 @@ +from rest_framework.response import Response +from rest_framework import status +from datetime import datetime, timedelta +from ...models import ( + Account, Member, Card, Site, Issue, Schedule, Flow, + get_meta_default, get_usage_default, Coupon +) +from ..ops.services import delete_site +from ..auth.services import create_or_update_account +from ..auth.serializers import AccountSerializer +from ...tasks import create_prospect +from cursion import settings +import stripe + + + + + + + + + +def stripe_setup(request: object) -> object: + """ + Creates or updates the Stripe Customer, Product, + Price, & Subscription associated with the passed + "user" and `Account` + + Args: + 'name' : 'free', 'cloud', 'selfhost', 'enterprise' (REQUIRED) + 'interval' : 'month' or 'year' (REQUIRED) + 'price_amount' : 1000 == $10 (REQUIRED) + 'task_amount' : 1000 == $10 (REQUIRED) + 'sites_allowed' : total # `Sites` per `Account` (REQUIRED) + 'pages_allowed' : total # `Pages` per `Site` (REQUIRED) + 'schedules_allowed' : total # `Schedules` per `Account` (REQUIRED) + 'retention_days' : total # days to keep data (REQUIRED) + 'caseruns_allowed' : total # of CaseRuns per `Account` per month (OPTIONAL) + 'scans_allowed' : total # of `Scans` per `Account` per month (OPTIONAL) + 'tests_allowed' : total # of `Tests` per `Account` per month (OPTIONAL) + 'caseruns_allowed' : total # of `CaseRuns` per `Account` per month (OPTIONAL) + 'flowruns_allowed' : total # of `FlowRuns` per `Account` per month (OPTIONAL) + 'nodes_allowed' : total # of `nodes` per `Flow` per month (OPTIONAL) + 'conditions_allowed' : total # of `conditons` per `Flow` (OPTIONAL) + 'meta' : any extra data for the account (OPTIONAL) + + + Returns: + 'subscription_id' : Stripe subscription id, + 'client_secret' : Stripe subscription client_secret, + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get request data + name = request.data.get('name') + interval = request.data.get('interval', 'month') # month or year + price_amount = int(request.data.get('price_amount')) + task_amount = int(request.data.get('task_amount')) + sites_allowed = int(request.data.get('sites_allowed')) + pages_allowed = int(request.data.get('pages_allowed')) + schedules_allowed = int(request.data.get('schedules_allowed')) + retention_days = int(request.data.get('retention_days')) + scans_allowed = int(request.data.get('scans_allowed')) + tests_allowed = int(request.data.get('tests_allowed')) + caseruns_allowed = int(request.data.get('caseruns_allowed')) + flowruns_allowed = int(request.data.get('flowruns_allowed')) + nodes_allowed = int(request.data.get('nodes_allowed')) + conditions_allowed = int(request.data.get('conditions_allowed')) + meta = request.data.get('meta', get_meta_default()) + + # get user + user = request.user + + # set defaults + initial_call = True + client_secret = None + default_product = None + default_price = None + task_product = None + task_price = None + prices = [] + + # build Stripe Default Product name + default_product_name = f'{name.capitalize()}' + + # get account + account = Account.objects.get(user=user) + + # get cursion task meter + meters = stripe.billing.Meter.list() + meter = meters['data'][0] + + # create new Stripe Customer & Product + if account.cust_id is None: + default_product = stripe.Product.create(name=default_product_name) + customer = stripe.Customer.create( + email = request.user.email, + name = f'{user.first_name} {user.last_name}' + ) + + # update existing Stripe Customer & Product + if account.cust_id is not None: + initial_call = False + default_product = stripe.Product.modify(account.product_id, name=default_product_name) + customer = stripe.Customer.retrieve(account.cust_id) + + # create new Stripe Default Price for + default_price = stripe.Price.create( + product = default_product.id, + unit_amount = price_amount, + currency = 'usd', + recurring = {'interval': interval,}, + ) + + # add to prices + prices.append(default_price) + + # create new Stripe Task Product & Price for CLOUD (Team & Business) + if name in ['cloud', 'team', 'business']: + + # create task product + task_product = stripe.Product.create(name='Tasks') + + # create task price + task_price = stripe.Price.create( + product = task_product.id, + unit_amount = task_amount, + currency = 'usd', + billing_scheme = 'per_unit', + recurring = { + 'usage_type' : 'metered', + 'interval' : 'month', + 'meter' : meter['id'] + }, + ) + + # add to prices + prices.append(task_price) + + # create new Stripe Subscription if none exists + if account.sub_id is None: + + # build items + items = [] + for price in prices: + items.append({ + 'price': price.id + }) + + # create subscription + subscription = stripe.Subscription.create( + customer = customer.id, + items = items, + payment_behavior = 'default_incomplete', + expand = ['latest_invoice.payment_intent'], + # trial_period_days = 7, + ) + + # update existing Stripe Subscription + if account.sub_id is not None: + + # get subscription + sub = stripe.Subscription.retrieve(account.sub_id) + + # build items + items = [] + i = 0 + for price in prices: + items.append({ + 'id' : sub['items']['data'][i].id, + 'price' : price.id + }) + i += 1 + + # updating price defaults and archiving old default_price + stripe.Product.modify(default_product.id, default_price=default_price,) + stripe.Price.modify(account.price_id, active=False) + + # get old task_price if available + if not task_price: + for item in sub['items']['data']: + if item['price']['recurring']['usage_type'] == 'metered': + task_price_id = item['price']['id'] + + # archive old task price + stripe.Price.modify(task_price_id, active=False) + + # update subscription + subscription = stripe.Subscription.modify( + sub.id, + cancel_at_period_end = False, + pause_collection = '', + proration_behavior = 'create_prorations', + items = items, + expand = ['latest_invoice.payment_intent'], + ) + + # update `Account` with new Stripe info + create_or_update_account( + user = user.id, + id = account.id, + type = name, + cust_id = customer.id, + sub_id = subscription.id, + product_id = default_product.id, + price_id = default_price.id, + price_amount = price_amount, + interval = interval, + sites_allowed = sites_allowed, + pages_allowed = pages_allowed, + schedules_allowed = schedules_allowed, + retention_days = retention_days, + scans_allowed = scans_allowed, + tests_allowed = tests_allowed, + caseruns_allowed = caseruns_allowed, + flowruns_allowed = flowruns_allowed, + nodes_allowed = nodes_allowed, + conditions_allowed = conditions_allowed, + meta = meta + ) + + # get client_secret from Stripe + # Subscription if Sub is new (i.e. initial_call == True) + if initial_call: + client_secret = subscription.latest_invoice.payment_intent.client_secret + + # format and return + data = { + 'subscription_id' : subscription.id, + 'client_secret' : client_secret, + } + return Response(data, status=status.HTTP_200_OK) + + + + +def stripe_complete(request: object) -> object: + """ + Confirms the Stripe Payment intent after user + enters CC details on Cursion.client - Also updates + `Account` payment method. + + Args: + 'payment_method' : stripe payment method id from client (REQUIRED) + + Returns: + `Account` HTTP Response object + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get request data + user = request.user + account = Account.objects.get(user=user) + pay_method_id = request.data['payment_method'] + + # get Stripe PaymentMethod object + pay_method = stripe.PaymentMethod.retrieve(pay_method_id) + + # create new `Card` if none exists + if Card.objects.filter(account=account).exists(): + + # attached Stripe Customer to existing + # Stripe PaymentMethod + stripe.PaymentMethod.attach( + pay_method_id, + customer=account.cust_id, + ) + + # update Stripe Customer + stripe.Customer.modify( + account.cust_id, + invoice_settings={ + 'default_payment_method': pay_method.id, + } + ) + + # update Stripe Subscription + stripe.Subscription.modify( + account.sub_id, + default_payment_method=pay_method.id + ) + + # update `Card` object + Card.objects.filter(account=account).update( + user = user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + + else: + # update Stripe Subscription with + # new payment method + stripe.Subscription.modify( + account.sub_id, + default_payment_method=pay_method_id + ) + + # create new `Card` object + Card.objects.create( + user = user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + + # update account activation + account.active = True + account.save() + + # update prospect + queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + create_prospect.apply_async(kwargs={'user_email': str(user.email)}, queue=queue, routing_key=queue) + + # serialize and return + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + return Response(data, status=status.HTTP_200_OK) + + + + +def calc_price(account: object=None) -> int: + """ + Calculates a `price` based on `Account.sites_allowed` + and any `Account.meta.coupon` data. + + Args: + 'account': (REQUIRED) + + Returns: + 'price_amount' + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get sites_allowed + sites_allowed = account.usage['sites_allowed'] + + # get account coupon + discount = 0 + if account.meta.get('coupon'): + discount = account.meta['coupon']['discount'] + + # calculate + price = ( + ( + (54.444 * (sites_allowed ** 0.4764)) + ) * 100 + ) + + # apply discount + price = price - (price * discount) + + # update for interval + price = round(price if account.interval == 'month' else (price * 10)) + + # return price + return int(price) + + + + +def get_stripe_hosted_url(request: object=None) -> object: + """ + Creates either a new 'Stripe Checkout Session' + (allows customer to subscribe), or a 'Stripe Customer + Portal Session' (allows customer to manage existing subscription). + Either session type with return a Stripe redirect url + + Args: + 'request' : (REQUIRED) + + Returns: + 'stripe_url': + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get account + user = request.user + account = Account.objects.get(user=user) + + # set default url + stripe_url = None + + # create Product, Price, & Checkout Session + if account.cust_id is None: + + # build product + product_name = f'Enterprise' + product = stripe.Product.create(name=product_name) + + # calc price_amount + price_amount = calc_price(account=account) + + # create new Stripe Price + price = stripe.Price.create( + product=product.id, + unit_amount=price_amount, + currency='usd', + recurring={'interval': account.interval,}, + ) + + # create Checkout Session + checkout_session = stripe.checkout.Session.create( + line_items=[ + { + 'price': price.id, + 'quantity': 1, + }, + ], + mode='subscription', + success_url=f'{settings.CLIENT_URL_ROOT}/billing/update' + + '?success=true&session_id={CHECKOUT_SESSION_ID}', + cancel_url=f'{settings.CLIENT_URL_ROOT}/billing', + ) + + # setting stripe_url + stripe_url = checkout_session.url + + # create Portal Session + if account.cust_id: + portal_session = stripe.billing_portal.Session.create( + customer=account.cust_id, + return_url=f'{settings.CLIENT_URL_ROOT}/billing/update', + ) + + # setting stripe_url + stripe_url = portal_session.url + + # return response + data = {'stripe_url': stripe_url} + return Response(data, status=status.HTTP_200_OK) + + + + +def update_account_with_stripe_redirect(request: object=None) -> object: + """ + Updates `Account` with new sub data from stripe redirect + + Args: + 'request' : (REQUIRED) + + Returns: + HTTP Response object + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get account + account = Account.objects.get(user=request.user) + cust_id = account.cust_id + sub_id = account.sub_id + + # try to get session_id + session_id = request.query_params.get('session_id') + + # if session_id - get customer, subscription + if session_id: + session = stripe.checkout.Session.retrieve( + session_id + ) + cust_id = session.customer + sub_id = session.subscription + + # get current stripe sub object + sub = stripe.Subscription.retrieve( + sub_id + ) + + # get stripe product & price info + plan = sub['items']['data'][0]['plan'] + product_id = plan['product'] + price_id = plan['id'] + price_amount = plan['amount'] + interval = plan['interval'] + + # setting Account.active + active = False if (sub['canceled_at'] or sub['pause_collection']) else True + + # get billing method info + pay_method_id = sub.default_payment_method + pay_method = stripe.PaymentMethod.retrieve( + pay_method_id + ) + + # create or update Account card + if not Card.objects.filter(account=account).exists(): + Card.objects.create( + user = request.user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + else: + Card.objects.filter(account=account).update( + user = request.user, + account = account, + pay_method_id = pay_method.id, + brand = pay_method.card.brand, + exp_year = pay_method.card.exp_year, + exp_month = pay_method.card.exp_month, + last_four = pay_method.card.last4 + ) + + # update `Account` with new Stripe info + create_or_update_account( + user = request.user.id, + id = account.id, + cust_id = cust_id, + sub_id = sub_id, + product_id = product_id, + price_id = price_id, + price_amount = price_amount, + interval = interval, + ) + + # starting account data deletion + if not active: + cancel_subscription(account=account) + + # serialize and return + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + return Response(data, status=status.HTTP_200_OK) + + + + +def get_billing_info(request: object) -> object: + """ + Gets the `Card`, `Account`, and slack info associated + with the passed "user". + + Args: + 'request' : (REQUIRED) + + Returns: + HTTP Response object + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # check is member exists + if not Member.objects.filter(user=request.user).exists(): + data = {'reason': 'member not found'} + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # set defaults + card = None + estimated_cost = None + + # get current task usage overages if cloud + if account.type in ['cloud', 'team', 'business']: + + task_count = 0 + task_items = ['caseruns', 'scans', 'tests'] + + for item in task_items: + overage = int(account.usage[item]) - int(account.usage[f'{item}_allowed']) + if overage > 0: + task_count += overage + + # calc current estimated costs + estimated_cost = round( + (account.price_amount) + + ( + (10 - (10 * account.meta['coupon']['discount'])) + * task_count + ) + ) + + # build plan + plan = { + 'name' : account.type, + 'active' : account.active, + 'price_amount' : account.price_amount, + 'interval' : account.interval, + 'usage' : account.usage, + 'meta' : account.meta, + 'estimated_cost' : estimated_cost + } + + # get `Card` info if exists + if Card.objects.filter(account=account).exists(): + _card = Card.objects.get(account=account) + card = { + 'brand' : _card.brand, + 'exp_year' : _card.exp_year, + 'exp_month' : _card.exp_month, + 'last_four' : _card.last_four, + } + + # format billing info + data = { + 'card': card, + 'plan': plan + } + + # return data + return Response(data, status=status.HTTP_200_OK) + + + + +def account_activation(request: object) -> object: + """ + Pauses or Activates the `Account` and billing + for the associated Stripe Subscription. + + Args: + 'request' : (REQUIRED) + + Returns: + Account` HTTP Response object + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get user's Account + account = Account.objects.get(user=request.user) + + # setting default + active = None + + # pause billing & `Account` + if account.active == True: + stripe.Subscription.modify( + account.sub_id, + pause_collection={ + 'behavior': 'mark_uncollectible', + }, + ) + active = False + + # activate billing & `Account` + else: + stripe.Subscription.modify( + account.sub_id, + pause_collection='', + ) + active = True + + # save updates + account.active = active + account.save() + + # serialize and return + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + return Response(data, status=status.HTTP_200_OK) + + + + +def cancel_subscription(request: object=None, account: object=None) -> object: + """ + Cancels the Stripe Subscription associated with the + passed "user" and reverts the `Account` to a "free" plan + + Args: + 'request': object (OPTIONAL) + 'account': object (OPTIONAL) + + Returns: + Account` HTTP Response object or Bool `true` + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get user's account + if request is not None: + user = request.user + account = Account.objects.get(user=user) + + # update billing if accout is active + if account.active == True: + + # canceling Stripe Subscription billing + try: + stripe.Subscription.cancel( + account.sub_id, + ) + except Exception as e: + print(e) + + # update Account plan + account.type = 'free' + account.interval = 'month' + account.price_amount = 0 + account.cust_id = None + account.sub_id = None + account.product_id = None + account.price_id = None + account.price_amount = None + account.usage = get_usage_default() + account.meta = get_meta_default() + + # save Account + account.save() + + # update user's card + card = Card.objects.get(account=account) + card.delete() + + # remove sites + for site in Site.objects.filter(account=account): + delete_site(id=site.id, user=user) + + # remove flows + for flow in Flow.objects.filter(account=account): + flow.delete() + + # remove issues + for issue in Issue.objects.filter(account=account): + issue.delete() + + # remove schedules + for schedule in Schedule.objects.filter(account=account): + schedule.delete() + + # serialize and return + if request is not None: + serializer_context = {'request': request,} + serialized = AccountSerializer(account, context=serializer_context) + data = serialized.data + return Response(data, status=status.HTTP_200_OK) + else: + return True + + + + +def get_stripe_invoices(request: object) -> object: + """ + Gets a list of Stripe Invoice objects associated with the + passed "user" `Account` + + Args: + 'request': object + + Returns: + 'has_more': true if more than 10 + 'data': of invoice objects + """ + + # init Stripe client + stripe.api_key = settings.STRIPE_PRIVATE + + # get user's account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # setting defaults + data = {"message": "no Account found"} + i_list = [] + + # check that Account has a Stripe Customer + if account.cust_id is not None: + + # retrieve Stripe Invoices + invoice_body = stripe.Invoice.list( + customer=account.cust_id, + ) + + # add to invoices list + invoices = [i for i in invoice_body.data] + + # build list of Stripe Invice objects + for invoice in invoices: + + # setting defaults + items = [] + product_name = None + interval = None + + # create line items + for item in invoice['lines']['data']: + + # add item data + items.append({ + 'amount': item['amount'], + 'description': item['description'], + 'period_start': item['period']['start'], + 'period_end': item['period']['end'], + 'quantity': item['quantity'], + 'proration': item['proration'], + 'unit_amount': item['price']['unit_amount'] + }) + + # getting product name and interval if item + # is not proration + if not item['proration']: + # get product_name + if 'cloud' in item['description'].lower(): + product_name = 'Cloud' + if 'team' in item['description'].lower(): + product_name = 'Team' + if 'business' in item['description'].lower(): + product_name = 'Business' + if 'selfhost' in item['description'].lower(): + product_name = 'Self Host' + if 'license' in item['description'].lower(): + product_name = 'License' + if 'manage' in item['description'].lower(): + product_name = 'Manage' + if 'enterprise' in item['description'].lower(): + product_name = 'Enterprise' + # get interval + interval = item['plan']['interval'] + + # get end_date + period_start = datetime.fromtimestamp(invoice.period_start) + new = period_start + timedelta(days=30 if interval == 'month' else 365) + period_end = int(new.timestamp()) + + i_list.append({ + 'id': invoice.id, + 'status': invoice.status, + 'subtotal': invoice.subtotal, + 'subtotal_excluding_tax': invoice.subtotal_excluding_tax, + 'price_amount': invoice.amount_paid, + 'created': invoice.created, + 'due_date': invoice.due_date, + 'customer_email': invoice.customer_email, + 'customer_name': invoice.customer_name, + 'product_name': product_name, + 'invoice_pdf': invoice.invoice_pdf, + 'number': invoice.number, + 'period_start': invoice.period_start, + 'period_end': period_end, + 'items': items + }) + + # format response + data = { + 'has_more': invoice_body.has_more, + 'data': i_list + } + + # return response + return Response(data, status=status.HTTP_200_OK) + + + + +### ------ Begin Coupon Services ------ ### + + + + +def check_coupon(request: object) -> object: + """ + Checks the passed 'query' against any existing + `Coupon.codes`. If found, returns "success=True" + and the whole `Coupon` object + + Args: { + 'request' : (REQUIRED) + + Returns: HTTP Response of serialized `Coupon` objects + """ + + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # get code + code = request.query_params.get('code') + + # defaults + coupon = None + success = False + + # check code against Coupons + if Coupon.objects.filter(code=code, status='active').exists(): + + # get coupon object + coup = Coupon.objects.get(code=code) + success = True + coupon = { + 'id': str(coup.id), + 'code': str(coup.code), + 'discount': float(coup.discount), + } + + # return + data = { + 'success': success, + 'coupon': coupon + } + + # return response + return Response(data, status=status.HTTP_200_OK) + + + + diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py index c39951f6..690d9711 100644 --- a/app/api/v1/billing/urls.py +++ b/app/api/v1/billing/urls.py @@ -4,15 +4,17 @@ -urlpatterns = [ - path('create-customer', views.CreateCustomer.as_view(), name='create_customer'), - path('create-product', views.CreateProduct.as_view(), name='create_product'), - path('create-price', views.CreatePrice.as_view(), name='create_price'), - path('create-subscription', views.CreateSubscription.as_view(), name='create_subscription'), - path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'), - path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'), - path('stripe-key', views.StripeKey.as_view(), name='stripe_key'), - path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'), - path('account-activation', views.AccountActivation.as_view(), name='account_activation') + +urlpatterns = [ + path('stripe/key', views.StripeKey.as_view(), name='stripe_key'), + path('invoices', views.StripeInvoice.as_view(), name='stripe_invoices'), + path('info', views.BillingInfo.as_view(), name='billing_info'), + path('subscription/setup', views.SubscriptionSetup.as_view(), name='subscription_setup'), + path('subscription/complete', views.SubscriptionComplete.as_view(), name='subscription_complete'), + path('subscription/cancel', views.SubscriptionCancel.as_view(), name='subscription_cancel'), + path('subscription/update', views.SubscriptionUpdate.as_view(), name='subscription_update'), + path('subscription/url', views.SubscriptionUrl.as_view(), name='subscription_url'), + path('account/activation', views.AccountActivation.as_view(), name='account_activation'), + path('coupon/search', views.Coupon.as_view(), name='coupon') ] diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py index 4ed57c74..073dc209 100644 --- a/app/api/v1/billing/views.py +++ b/app/api/v1/billing/views.py @@ -1,383 +1,122 @@ from rest_framework.response import Response -from rest_framework.permissions import AllowAny +from rest_framework.permissions import IsAuthenticated from rest_framework.views import APIView from rest_framework import status -from django.contrib.auth.models import User -from django.core import serializers -from django.forms.models import model_to_dict -from ...models import Account, Card -from ..auth.services import create_or_update_account -from datetime import timedelta, datetime -from scanerr import settings -import os, stripe, json +from .services import * +from cursion import settings + + class StripeKey(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post',] - def post(self, request): - key = settings.STRIPE_PUBLIC - data = {'key': key,} + def post(self, request): + data = {'key': settings.STRIPE_PUBLIC,} return Response(data, status=status.HTTP_200_OK) -class CreateCustomer(APIView): - permission_classes = (AllowAny,) +class SubscriptionSetup(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): - stripe.api_key = settings.STRIPE_PRIVATE - customer = stripe.Customer.create(email=request.user.email) - - account = Account.objects.create( - user=request.user, - cust_id=customer.id - ) + response = stripe_setup(request) + return response - data = customer.__dict__ - - return Response(data, status=status.HTTP_200_OK) -class CreateProduct(APIView): - permission_classes = (AllowAny,) +class SubscriptionComplete(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): - name = request.data['name'] - stripe.api_key = settings.STRIPE_PRIVATE - product = stripe.Product.create(name=name) - - account = Account.objects.get(user=request.user) - account.product_id = product.id - account.save() - - data = product.__dict__ - - return Response(data, status=status.HTTP_200_OK) + response = stripe_complete(request) + return response -class CreatePrice(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - def post(self, request): - account = Account.objects.get(user=request.user) - price_amount = float(request.data['price_amount']) - stripe.api_key = settings.STRIPE_PRIVATE - price = stripe.Price.create( - product=account.product_id, - unit_amount=price_amount, - currency='usd', - recurring={ - 'interval': 'month', - 'trial_period_days': 7, - }, - ) - - account.price_id = price.id - account.save() +class SubscriptionUrl(APIView): + permission_classes = (IsAuthenticated,) + https_method_names = ['get',] - data = price.__dict__ - - return Response(data, status=status.HTTP_200_OK) + def get(self, request): + response = get_stripe_hosted_url(request) + return response -class CreateSubscription(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - def post(self, request): - stripe.api_key = settings.STRIPE_PRIVATE - account = Account.objects.get(user=request.user) - subscription = stripe.Subscription.create( - customer=account.cust_id, - items=[{ - 'price': account.price_id, - }], - payment_behavior='default_incomplete', - expand=['latest_invoice.payment_intent'], - ) - - account.sub_id = subscription.id - account.save() - data = { - 'subscription_id' : subscription.id, - 'client_secret' : subscription.latest_invoice.payment_intent.client_secret - } - - return Response(data, status=status.HTTP_200_OK) +class SubscriptionUpdate(APIView): + permission_classes = (IsAuthenticated,) + https_method_names = ['get',] + def get(self, request): + response = update_account_with_stripe_redirect(request) + return response -class CompleteSubscription(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - def post(self, request): - stripe.api_key = settings.STRIPE_PRIVATE - account = Account.objects.get(user=request.user) - pay_method_id = request.data['payment_method'] - if Card.objects.filter(account=account).exists(): - pay_method = stripe.PaymentMethod.retrieve(pay_method_id) - - stripe.PaymentMethod.attach( - pay_method_id, - customer=account.cust_id, - ) - - stripe.Customer.modify( - account.cust_id, - invoice_settings={ - 'default_payment_method': pay_method.id, - } - ) - - stripe.Subscription.modify( - account.sub_id, - default_payment_method=pay_method.id - ) - - Card.objects.filter(account=account).update( - user = request.user, - account = account, - pay_method_id = pay_method.id, - brand = pay_method.card.brand, - exp_year = pay_method.card.exp_year, - exp_month = pay_method.card.exp_month, - last_four = pay_method.card.last4 - ) - - else: - pay_method = stripe.PaymentMethod.retrieve(pay_method_id) - - stripe.Subscription.modify( - account.sub_id, - default_payment_method=pay_method.id - ) - - Card.objects.create( - user = request.user, - account = account, - pay_method_id = pay_method.id, - brand = pay_method.card.brand, - exp_year = pay_method.card.exp_year, - exp_month = pay_method.card.exp_month, - last_four = pay_method.card.last4 - - ) - - card = Card.objects.get(account=account) - account.active = True - account.save() - - data = { - 'card': { - 'brand': card.brand, - 'exp_year': card.exp_year, - 'exp_month': card.exp_month, - 'last_four': card.last_four, - }, - 'plan': { - 'name': account.type, - 'active': account.active, - 'slack': { - 'slack_name': account.slack['slack_name'], - 'bot_user_id': account.slack['bot_user_id'], - 'slack_team_id': account.slack['slack_team_id'], - 'bot_access_token': account.slack['bot_access_token'], - 'slack_channel_id': account.slack['slack_channel_id'], - 'slack_channel_name': account.slack['slack_channel_name'], - } - }, - } +class BillingInfo(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + def post(self, request): + response = get_billing_info(request) + return response - return Response(data, status=status.HTTP_200_OK) -class SetupSubscription(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] +class AccountActivation(APIView): + permission_classes = (IsAuthenticated,) + https_method_names = ['post',] - def post(self, request): - stripe.api_key = settings.STRIPE_PRIVATE - user = request.user - name = request.data.get('name') - product_name = str(user.email + '_' + str(user.id) + '_' + name) - price_amount = int(request.data.get('price_amount')) - max_sites = int(request.data.get('max_sites')) - - if not Account.objects.filter(user=user).exists(): - create_or_update_account( - user=user, - type=name, - max_sites=max_sites, - ) - - account = Account.objects.get(user=user) - - if account.cust_id is None: - product = stripe.Product.create(name=product_name) - customer = stripe.Customer.create(email=request.user.email) - - if account.cust_id is not None: - product = stripe.Product.modify(account.product_id, name=product_name) - customer = stripe.Customer.retrieve(account.cust_id) - - price = stripe.Price.create( - product=product.id, - unit_amount=price_amount, - currency='usd', - recurring={'interval': 'month',}, - ) - - if account.sub_id is None: - subscription = stripe.Subscription.create( - customer=customer.id, - items=[{ - 'price': price.id, - }], - payment_behavior='default_incomplete', - expand=['latest_invoice.payment_intent'], - # trial_period_days=7, - ) - - if account.sub_id is not None: - sub = stripe.Subscription.retrieve(account.sub_id) - subscription = stripe.Subscription.modify( - sub.id, - cancel_at_period_end=False, - proration_behavior='create_prorations', - items=[{ - 'id': sub['items']['data'][0].id, - 'price': price.id, - }], - expand=['latest_invoice.payment_intent'], - ) + def post(self, request): + response = account_activation(request) + return response - # updating price defaults and archiving old price - stripe.Product.modify(product.id, default_price=price,) - stripe.Price.modify(account.price_id, active=False) - Account.objects.filter(user=user).update( - type = name, - cust_id = customer.id, - sub_id = subscription.id, - product_id = product.id, - price_id = price.id, - max_sites = max_sites, - ) - data = { - 'subscription_id' : subscription.id, - 'client_secret' : subscription.latest_invoice.payment_intent.client_secret, - } +class SubscriptionCancel(APIView): + permission_classes = (IsAuthenticated,) + https_method_names = ['post',] + def post(self, request): + response = cancel_subscription(request) + return response + - return Response(data, status=status.HTTP_200_OK) +class StripeInvoice(APIView): + permission_classes = (IsAuthenticated,) + https_method_names = ['get',] + def get(self, request): + response = get_stripe_invoices(request) + return response -class GetBillingInfo(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - def post(self, request): - user = request.user - if Account.objects.filter(user=user).exists(): - card = Card.objects.get(user=user) - account = Account.objects.get(user=user) - - data = { - 'card': { - 'brand': card.brand, - 'exp_year': card.exp_year, - 'exp_month': card.exp_month, - 'last_four': card.last_four, - }, - 'plan': { - 'name': account.type, - 'active': account.active, - 'slack': { - 'slack_name': account.slack['slack_name'], - 'bot_user_id': account.slack['bot_user_id'], - 'slack_team_id': account.slack['slack_team_id'], - 'bot_access_token': account.slack['bot_access_token'], - 'slack_channel_id': account.slack['slack_channel_id'], - 'slack_channel_name': account.slack['slack_channel_name'], - } - }, - } - - return Response(data, status=status.HTTP_200_OK) - - else: - return Response(status=status.HTTP_404_NOT_FOUND) +class Coupon(APIView): + permission_classes = (IsAuthenticated,) + https_method_names = ['get',] + def get(self, request): + response = check_coupon(request) + return response -class AccountActivation(APIView): - permission_classes = (AllowAny,) - https_method_names = ['post',] - def post(self, request): - account = Account.objects.get(user=request.user) - stripe.api_key = settings.STRIPE_PRIVATE - - if account.active == True: - stripe.Subscription.modify( - account.sub_id, - pause_collection={ - 'behavior': 'mark_uncollectible', - }, - ) - account.active = False - account.save() - else: - stripe.Subscription.modify( - account.sub_id, - pause_collection='', - ) - account.active = True - account.save() - - card = Card.objects.get(account=account) - - data = { - 'card': { - 'brand': card.brand, - 'exp_year': card.exp_year, - 'exp_month': card.exp_month, - 'last_four': card.last_four, - }, - 'plan': { - 'name': account.type, - 'active': account.active, - 'slack': { - 'slack_name': account.slack['slack_name'], - 'bot_user_id': account.slack['bot_user_id'], - 'slack_team_id': account.slack['slack_team_id'], - 'bot_access_token': account.slack['bot_access_token'], - 'slack_channel_id': account.slack['slack_channel_id'], - 'slack_channel_name': account.slack['slack_channel_name'], - }, - }, - } - return Response(data, status=status.HTTP_200_OK) - \ No newline at end of file diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py index a74996b7..12505bcd 100644 --- a/app/api/v1/ops/serializers.py +++ b/app/api/v1/ops/serializers.py @@ -2,11 +2,17 @@ from rest_framework import serializers from rest_framework.fields import UUIDField + + + + + kwargs = { 'allow_null': False, 'read_only': True, 'pk_field': UUIDField(format='hex_verbose') - } +} + @@ -22,6 +28,7 @@ class Meta: + class ProcessSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) @@ -29,11 +36,39 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Process fields = ['id', 'site', 'type', 'time_created', 'time_completed', - 'successful', 'info_url', 'progress', + 'success', 'info_url', 'progress', 'info', 'exception', 'object_id' + ] + + + + +class SecretSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Secret + fields = ['id', 'account', 'user', 'time_created', 'name', + ] + + + + +class ChatSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Chat + fields = ['id', 'user', 'account', 'time_created', + 'status', 'messages' ] + class SiteSerializer(serializers.HyperlinkedModelSerializer): user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) @@ -42,25 +77,46 @@ class SiteSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Site fields = ['id', 'user', 'site_url', 'time_created', 'info', + 'tags', 'account', 'time_crawl_started', 'time_crawl_completed', + ] + + + + +class PageSerializer(serializers.HyperlinkedModelSerializer): + user = serializers.ReadOnlyField(source='user.username') + id = serializers.PrimaryKeyRelatedField(**kwargs) + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + + class Meta: + model = Page + fields = ['id', 'user', 'site', 'page_url', 'time_created', 'info', 'tags', 'account', ] + + class ScanSerializer(serializers.HyperlinkedModelSerializer): site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) + page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs) paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs) id = serializers.PrimaryKeyRelatedField(**kwargs) class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', + fields = ['id', 'site', 'page', 'paired_scan', 'time_created', 'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', - 'images', 'configs', 'tags', 'type', + 'images', 'configs', 'tags', 'type', 'score', ] + + class SmallScanSerializer(serializers.HyperlinkedModelSerializer): site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs) + page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs) paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs) lighthouse = serializers.SerializerMethodField() yellowlab = serializers.SerializerMethodField() @@ -74,64 +130,73 @@ def get_yellowlab(self, obj): class Meta: model = Scan - fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', - 'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', + fields = ['id', 'site', 'page', 'paired_scan', 'time_created', 'logs', + 'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', 'score', ] + + class TestSerializer(serializers.HyperlinkedModelSerializer): - site = serializers.PrimaryKeyRelatedField(**kwargs) - pre_scan = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs) + pre_scan = serializers.PrimaryKeyRelatedField(source='pre_scan.id', **kwargs) post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs) id = serializers.PrimaryKeyRelatedField(**kwargs) class Meta: model = Test - fields = ['id', 'site', 'time_created', 'time_completed', + fields = ['id', 'site', 'page', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta', - 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', - 'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores', + 'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', 'threshold', + 'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores', 'status', ] + + class SmallTestSerializer(serializers.HyperlinkedModelSerializer): - site = serializers.PrimaryKeyRelatedField(**kwargs) - pre_scan = serializers.PrimaryKeyRelatedField(**kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs) + pre_scan = serializers.PrimaryKeyRelatedField(source='pre_scan.id', **kwargs) post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs) id = serializers.PrimaryKeyRelatedField(**kwargs) class Meta: model = Test - fields = ['id', 'site', 'time_created', 'time_completed', + fields = ['id', 'site', 'page', 'time_created', 'time_completed', 'pre_scan', 'post_scan', 'score', 'lighthouse_delta', - 'yellowlab_delta', 'tags', 'component_scores', + 'yellowlab_delta', 'tags', 'component_scores', 'threshold', 'status', ] + + class ScheduleSerializer(serializers.HyperlinkedModelSerializer): - site = serializers.PrimaryKeyRelatedField(**kwargs) user = serializers.ReadOnlyField(source='user.username') id = serializers.PrimaryKeyRelatedField(**kwargs) - automation = serializers.PrimaryKeyRelatedField(**kwargs) + alert = serializers.PrimaryKeyRelatedField(**kwargs) account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: model = Schedule - fields = ['id', 'site', 'time_created', 'user', 'task_type', + fields = ['id', 'time_created', 'user', 'task_type', 'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id', - 'periodic_task_id', 'status', 'automation', 'extras', 'account', + 'periodic_task_id', 'status', 'alert', 'extras', 'account', + 'scope', 'resources', 'tags', 'time_last_run', ] -class AutomationSerializer(serializers.HyperlinkedModelSerializer): + +class AlertSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) schedule = serializers.PrimaryKeyRelatedField(**kwargs) user = serializers.ReadOnlyField(source='user.username') account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: - model = Automation + model = Alert fields = ['id', 'expressions', 'actions', 'user', 'schedule', 'time_created', 'name', 'account', ] @@ -158,16 +223,18 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) user = serializers.ReadOnlyField(source='user.username') account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) class Meta: model = Case - fields = ['id', 'name', 'user', 'steps', 'time_created', - 'tags', 'account', + fields = ['id', 'title', 'user', 'steps', 'time_created', + 'tags', 'account', 'site', 'type', 'site_url', 'processed' ] -class TestcaseSerializer(serializers.HyperlinkedModelSerializer): + +class CaseRunSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) @@ -175,13 +242,15 @@ class TestcaseSerializer(serializers.HyperlinkedModelSerializer): account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: - model = Testcase + model = CaseRun fields = ['id', 'site', 'user', 'time_created', 'time_completed', - 'steps', 'case', 'case_name', 'passed', 'configs', 'account', + 'steps', 'case', 'title', 'configs', 'account', 'status', ] -class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer): + + +class SmallCaseRunSerializer(serializers.HyperlinkedModelSerializer): id = serializers.PrimaryKeyRelatedField(**kwargs) site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) case = serializers.PrimaryKeyRelatedField(source='case.id', **kwargs) @@ -189,7 +258,71 @@ class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer): account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) class Meta: - model = Testcase + model = CaseRun fields = ['id', 'site', 'user', 'time_created', 'time_completed', - 'case', 'case_name', 'passed', 'configs', 'account', - ] \ No newline at end of file + 'case', 'title', 'configs', 'account', 'status', + ] + + + + +class IssueSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Issue + fields = ['id', 'time_created', 'trigger', 'account', 'title', + 'details', 'status', 'affected', 'labels' + ] + + + + +class FlowSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + + class Meta: + model = Flow + fields = ['id', 'user', 'account', 'time_created', 'title', + 'nodes', 'edges', 'time_last_run', + ] + + + + +class FlowRunSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + flow = serializers.PrimaryKeyRelatedField(source='flow.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + + class Meta: + model = FlowRun + fields = ['id', 'user', 'account', 'flow', 'time_created', 'title', + 'nodes', 'edges', 'status', 'time_completed', 'logs', 'site', 'configs' + ] + + + + +class SmallFlowRunSerializer(serializers.HyperlinkedModelSerializer): + id = serializers.PrimaryKeyRelatedField(**kwargs) + flow = serializers.PrimaryKeyRelatedField(source='flow.id', **kwargs) + user = serializers.ReadOnlyField(source='user.username') + account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs) + site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs) + + class Meta: + model = FlowRun + fields = ['id', 'user', 'account', 'flow', 'time_created', 'title', + 'status', 'time_completed', 'site', 'configs' + ] + + + + + diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py index da27dd42..a1586ffd 100644 --- a/app/api/v1/ops/services.py +++ b/app/api/v1/ops/services.py @@ -1,40 +1,64 @@ -import json, boto3, asyncio -from datetime import datetime from django.contrib.auth.models import User from django_celery_beat.models import CrontabSchedule, PeriodicTask -from ...models import * +from django.db.models import Q +from functools import reduce +from django.http import HttpResponse +from django.utils import timezone +from django.core.cache import cache +from rest_framework.pagination import LimitOffsetPagination from rest_framework.response import Response from rest_framework import status +from cryptography.fernet import Fernet +from cursion import celery +from redis import Redis +from redis.exceptions import RedisError +from cursion import settings from .serializers import * from ...tasks import * -from rest_framework.pagination import LimitOffsetPagination -from ...utils.scanner import Scanner as S -from ...utils.tester import Tester as T -from ...utils.image import Image as I +from ...models import * from ...utils.reporter import Reporter as R -from ...utils.wordpress import Wordpress as W -from ...utils.wordpress_p import Wordpress as W_P -from ...utils.caser import Caser +from ...utils.devices import devices +from ...utils.issuer import Issuer +from datetime import datetime, timedelta +import json, boto3, os, requests, uuid, secrets, operator +ON_DEMAND_QUEUE = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') -def record_api_call(request, data, status): +def record_api_call(request: object, data: dict, status: str) -> None: + """ + Records an request and resposne if + the request was sent with Token auth. + Creates a `Log` with the recorded info + + Args: + request : object, + data : dict, + status : str + + Returns: + None + """ + + # get auth type auth = request.headers.get('Authorization') + + # check if Token auth if auth.startswith('Token'): + # getting the request data if request.method == 'POST': request_data = request.data - elif request.method == 'GET': request_data = request.query_params - elif request.method == 'DELETE': request_data = request.query_params + # recording info log = Log.objects.create( user=request.user, path=request.path, @@ -43,178 +67,782 @@ def record_api_call(request, data, status): request_payload=request_data, response_payload=data ) + + return None + + + + +def decrement_resource(account: object, resource: str) -> None: + """ + Removes '1' from the resource total + + Args: + 'account' : + 'resource' : 'site', 'page', 'schedule' - return + Returns: + None + """ + # remove 1 from account.usage[{resource}] + account.usage[f'{resource}'] -= 1 + account.save() + # return None + return None -def check_account(request): - if Member.objects.filter(user=request.user).exists(): - member = Member.objects.get(user=request.user) - return member.account.active - else: + + + +def check_location(request: None, local: None) -> dict: + """ + Reroutes a request to a geo-specific + instance of Cursion Server. + + Args: + 'request': obj, + 'local' : str, + + Returns: + 'routed': bool (True if request was forwarded), + 'response': obj (HTTP response from forwarded request) + """ + + # set defaults + routed = False + response = None + + # checking if request was passed + if request: + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # get configs obj & location + configs = request.data.get('configs', account.configs) + location = local if local else configs.get('location', settings.LOCATION) + + # get path & build url + path = request.path + root = settings.API_URL_ROOT.lstrip('https://') + url = f'https://{location}-{root}{path}' + + # get authorization & build headers + auth = request.headers.get('Authorization') + headers = { + 'Content-Type': 'application/json', + 'Authorization': auth + } + + # check location and forward request + if location != settings.LOCATION: + routed = True + # send request + print(f'forwarding request to: {url}') + resp = requests.post( + url=url, + headers=headers, + data=json.dumps(request.data) + ) + # build response + response = HttpResponse( + content=resp.content, + status=resp.status_code, + headers=resp.headers + ) + + # return data + data = { + 'routed': routed, + 'response': response + } + return data + + + + +def check_permissions_and_usage( + member: object=None, + resource: str=None, + action: str='get', + id: str=None, + id_type: str=None, + url: str=None + ) -> dict: + """ + References Member.permissions to determine if + give action is allowed on given resource. + + Args: + 'member' : obj, (REQUIRED) + 'resource' : str, (REQUIRED) + 'action' : str, (OPTIONAL, 'get') + 'id' : str, (OPTIONAL) + 'id_type' : str, (OPTIONAL) + 'url' : str, (OPTIONAL) + + Returns: + 'allowed' : bool + 'error' : str + 'code': : str + 'status': : object + """ + + # get account from member + account = member.account + + # set default + allowed = True + error = 'not allowed' + code = '403' + _status = status.HTTP_403_FORBIDDEN + + # ignore site assoc checks on these resorces + ignore_list = ['alert', 'schedule', 'log', 'process', 'flow', 'secret', 'chat'] + + # check usage on these resources + usage_list = ['site', 'schedule', 'caserun', 'flowrun', 'scan', 'test'] + + # helper method to search permissions.sites + def site_in_sites(id) -> bool: + if len(member.permissions.get('sites', [])) == 0: + return True + for site in member.permissions.get('sites'): + if id == site['id']: + return True return False + # check action with permissions + if action not in member.permissions.get('actions'): + return { + 'allowed': False, + 'error': error, + 'code': code, + 'status': _status + } + -def create_site(request, delay=False): - site_url = request.data.get('site_url') - user = request.user - account = Member.objects.get(user=user).account - sites = Site.objects.filter(account=account) + # check resource with permissions + if resource not in member.permissions.get('resources'): + return { + 'allowed': False, + 'error': error, + 'code': code, + 'status': _status + } + + + # check id + if id and id_type: + + # create obj_str + obj_str = id_type.capitalize() + if 'run' in obj_str: + obj_str = obj_str.replace('run', 'Run') + + # retrieve obj + if id_type not in ['scan', 'test']: + objs = eval(f'{obj_str}.objects.filter(id="{id}", account__id="{account.id}")') + if id_type in ['scan', 'test']: + objs = eval(f'{obj_str}.objects.filter(id="{id}", site__account__id="{account.id}")') + + # return False if not found + if len(objs) == 0: + return { + 'allowed': False, + 'error': f'{resource} not found', + 'code': '404', + 'status': status.HTTP_404_NOT_FOUND + } + + + # check for site association + if (id and id_type) and id_type not in ignore_list: + + # special case for `Issue` + if id_type == 'issue': + affected_type = objs[0].affected.get('type') + + if affected_type == 'site': + # check site in permissions.sites + if not site_in_sites(objs[0].affected.get('id')): + return { + 'allowed': False, + 'error': error, + 'code': code, + 'status': _status + } + + if affected_type == 'page': + # check page.site in permissions.sites + try: + page = Page.objects.get(id=objs[0].affected.get('id')) + if not site_in_sites(str(page.site.id)): + return { + 'allowed': False, + 'error': error, + 'code': code, + 'status': _status + } + except: + return { + 'allowed': False, + 'error': f'{resource} not found', + 'code': '404', + 'status': status.HTTP_404_NOT_FOUND + } + + # check site in permissions.sites + elif id_type == 'site': + if not site_in_sites(str(objs[0].id)): + return { + 'allowed': False, + 'error': error, + 'code': code, + 'status': _status + } + + # check associated site in permissions.sites + else: + if objs[0].site: + if not site_in_sites(str(objs[0].site.id)): + return { + 'allowed': False, + 'error': error, + 'code': code, + 'status': _status + } + + + # handle special cases for site and page + if (resource == 'site' or resource == 'page') and account.user.username != 'admin': + # check existance + if url: + if eval(f'{resource.capitalize()}.objects.filter(account__id="{account.id}", {resource}_url="{url}").exists()'): + return { + 'allowed': False, + 'error': f'{resource} exists', + 'code': '409', + 'status': status.HTTP_409_CONFLICT + } + + # check usage for page only + if resource == 'page' and id_type == 'site' and action == 'add': + if account.usage['pages_allowed'] == Page.objects.filter(site__id=id).count(): + return { + 'allowed': False, + 'error': f'max pages reached', + 'code': '426', + 'status': status.HTTP_426_UPGRADE_REQUIRED + } + + + # check for cloud / enterprise plan + if (account.type == 'enterprise') and resource == 'site' and account.user.username != 'admin' : + + # add to sites_allowed only for enterprise and cloud plans + if action == 'add' and account.usage['sites_allowed'] == Site.objects.filter(account=account).count(): + account.usage['sites_allowed'] += 1 + account.usage['schedules_allowed'] += 1 + account.save() + + # update price for sub if enterprise + if account.type == 'enterprise': + update_sub_price.apply_async(kwargs={'account_id': str(account.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + + # check usage if action is 'add' + if action == 'add' and resource in usage_list and account.user.username != 'admin': + + # check if usage allows for 'add' + if (int(account.usage[f'{resource}s']) >= int(account.usage[f'{resource}s_allowed'])): + + # return UPGRADE_REQUIRED if not cloud + if account.type != 'cloud': + return { + 'allowed': False, + 'error': f'max {resource}s reached', + 'code': '426', + 'status': status.HTTP_426_UPGRADE_REQUIRED + } + + + # return True + return { + 'allowed': True, + 'error': None, + 'code': '201' if action == 'add' else '200', + 'status': status.HTTP_201_CREATED if action == 'add' else status.HTTP_200_OK + } + + + + +def retry_failed_tasks(request: object=None) -> object: + """ + Using `tasks.redeliver_failed_tasks()`, retries any + "failed" task that has not reached MAX_ATTEMPTS. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # init rety + redeliver_failed_tasks() + + # return response + data = {'message': 'redelivered failed tasks'} + response = Response(data, status=status.HTTP_200_OK) + return response - if site_url.endswith('/'): - site_url = site_url.rstrip('/') - if site_url is None or site_url == '': - data = {'reason': 'the site_url cannot be empty',} + +### ------ Begin Site Services ------ ### + + + + +def create_or_update_site(request: object=None) -> object: + """ + Creates a new `Site`, initiates a Crawl, initial `Scans` + for each added `Page`, and generates new `Cases`. + + Args: + request : object + + Returns: + HTTP Response object + """ + + # getting data + site_id = request.data.get('site_id') + site_url = request.data.get('site_url') + page_urls = request.data.get('page_urls') + onboarding = request.data.get('onboarding', None) + tags = request.data.get('tags', None) + configs = request.data.get('configs', None) + no_scan = request.data.get('no_scan', False) + + # gettting account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # checking if in onboarding flow + if onboarding is not None: + if str(onboarding).lower() == 'true': + onboarding = True + if str(onboarding).lower() == 'false': + onboarding = False + + # clean & check site url + if (site_url is None or site_url == '') and not site_id: + data = {'reason': 'the site_url cannot be empty'} record_api_call(request, data, '400') return Response(data, status=status.HTTP_400_BAD_REQUEST) + if site_url: + if site_url.endswith('/'): + site_url = site_url.rstrip('/') + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='site', + action='update' if site_id else 'add', + url=site_url, id=site_id + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # update site if site_id passed + if site_id: + + # get site & update data + site = Site.objects.get(id=site_id) + if tags is not None: + site.tags = tags + site.save() + + # serialize response and return + serialized = SiteSerializer(site, context={'request': request}) + record_api_call(request, serialized.data, '200') + response = Response(serialized.data, status=status.HTTP_200_OK) + return response - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # creating site if checks passed + site = Site.objects.create( + site_url=site_url, + user=user, + tags=tags, + account=account, + time_crawl_started=datetime.now() + ) + + # updated accounts usage + account.usage['sites'] += 1 + account.save() - if sites.count() >= account.max_sites: - data = {'reason': 'maximum number of sites reached',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # create process obj + process = Process.objects.create( + site=site, + type='case.generate', + account=account, + progress=1 + ) - if Site.objects.filter(site_url=site_url, user=user).exists(): - data = {'reason': 'site already exists',} - record_api_call(request, data, '409') - return Response(data, status=status.HTTP_409_CONFLICT) - else: - tags = request.data.get('tags', None) - configs = request.data.get('configs', None) - no_scan = request.data.get('no_scan', False) - site = Site.objects.create( - site_url=site_url, - user=user, - tags=tags, - account=account - ) + # auto gen Cases using bg_autocase_task + create_auto_cases_bg.apply_async( + kwargs={ + 'site_id': str(site.id), + 'process_id': str(process.id), + 'start_url': str(site.site_url), + 'configs': configs, + 'max_cases': 3, + 'max_layers': 8, + }, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # check if this is account's first site and onboarding = True + if Site.objects.filter(account=account).count() == 1 \ + and onboarding == True: + # send POST to landing/v1/ops/prospect + create_prospect.apply_async(kwargs={'user_email': str(user.email)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # check if scan requested + if no_scan == False: + + # adding pages passed in request + if page_urls is not None: + for url in page_urls: + if url.startswith(site.site_url): + # add new page + page = Page.objects.create( + site=site, + page_url=url, + user=site.user, + account=site.account, + ) + # create scan + create_scan( + page_id=page.id, + configs=configs, + user_id=request.user.id, + delay=True + ) + site.time_crawl_started = datetime.now() + site.time_crawl_completed = datetime.now() + site.info["latest_scan"]["time_created"] = str(datetime.now()) + site.save() + + # starting crawler and scans in background + else: + create_site_and_pages_bg.apply_async( + kwargs={'site_id': str(site.id), 'configs': configs}, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # serialize response and return + serialized = SiteSerializer(site, context={'request': request}) + record_api_call(request, serialized.data, '201') + response = Response(serialized.data, status=status.HTTP_201_CREATED) + return response - if not configs: - configs = { - 'window_size': '1920,1080', - 'interval': 5, - 'driver': 'selenium', - 'device': 'desktop', - 'mask_ids': None, - 'min_wait_time': 10, - 'max_wait_time': 60, - 'timeout': 300, - 'disable_animations': False - } - if no_scan == False: - if delay == True: - scan = Scan.objects.create( - site=site, - type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'], - configs=configs, - ) - # running scans in parallel - if 'html' or 'logs' or 'full' in types: - run_html_and_logs_bg.delay(scan_id=scan.id) - if 'lighthouse' or 'full' in types: - run_lighthouse_bg.delay(scan_id=scan.id) - if 'yellowlab' or 'full' in types: - run_yellowlab_bg.delay(scan_id=scan.id) - if 'vrt' or 'full' in types: - run_vrt_bg.delay(scan_id=scan.id) - # create_site_bg.delay(site.id, scan.id, configs) - site.info["latest_scan"]["id"] = str(scan.id) - site.info["latest_scan"]["time_created"] = str(scan.time_created) - site.save() - else: - S(site=site, configs=configs).first_scan() - - serializer_context = {'request': request,} - serialized = SiteSerializer(site, context=serializer_context) + + +def crawl_site(request: object=None, id: str=None, user: object=None) -> object: + """ + Initiates a new Crawl for the passed `Site`.id + + Args: + 'request' : object, + 'id' : str, + 'user' : object + + Returns: + HTTP Response object + """ + + # get user and account + if request: + user = request.user + + member = Member.objects.get(user=user) + account = member.account + configs = request.data.get('configs', None) + + # updating configs if None: + configs = account.configs if configs == None else configs + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='site', action='get', + id=id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # update site info + site = Site.objects.get(id=id) + site.time_crawl_completed = None + site.save() + + # starting crawl + crawl_site_bg.apply_async(kwargs={'site_id': str(site.id), 'configs': configs}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # serializing and returning + if request: + serializer_context = {'request': request} + serialized = SiteSerializer(site, context={'request': request}) data = serialized.data record_api_call(request, data, '201') response = Response(data, status=status.HTTP_201_CREATED) return response + return None +def get_sites(request: object=None) -> object: + """ + Get one or more `Sites` in paginated response + Args: + 'request': object -def get_sites(request): + Returns: + HTTP Response object + """ + + # getting request data site_id = request.query_params.get('site_id') + sort = request.query_params.getlist('sort') user = request.user - account = Member.objects.get(user=user).account + # getting account + member = Member.objects.get(user=user) + account = member.account + + # site-specific sorting dict + sorting_items = { + # accend + 'site': 'site_url', + 'time_created': 'time_created', + 'scan': 'info__latest_scan__score', + 'scan_completed': 'info__latest_scan__time_completed', + 'test': 'info__latest_test__score', + # decend + '-site': '-site_url', + '-time_created': '-time_created', + '-scan': '-info__latest_scan__score', + '-scan_completed': '-info__latest_scan__time_completed', + '-test': '-info__latest_test__score', + } + + # transform sort param + _sort = [] + for s in sort: + _sort.extend(s.split(',')) - if site_id != None: - - try: - site = Site.objects.get(id=site_id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # build order_by list (ignore invalid sort tokens) + ordering = [sorting_items[s] for s in _sort if s in sorting_items] + if not ordering: + ordering = ['-time_created'] + + # check if site_id was passed + if site_id: + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='site', action='get', id=site_id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - if site.account != account: - data = {'reason': 'retrieve a Site you do not own',} - return Response(data, status=status.HTTP_403_FORBIDDEN) - serializer_context = {'request': request,} - serialized = SiteSerializer(site, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + # get site if checks passed + site = Site.objects.get(id=site_id) + + # serialize single site response and return + serialized = SiteSerializer(site, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) - sites = Site.objects.filter(account=account).order_by('-time_created') + # getting all account assoicated sites + sites = Site.objects.filter(account=account).order_by(*ordering) + + # filter out all non permissioned sites + if len(member.permissions.get('sites', [])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + sites = sites.filter(id__in=id_list).order_by(*ordering) + + # serialize response and return paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(sites, request) - serializer_context = {'request': request,} - serialized = SiteSerializer(result_page, many=True, context=serializer_context) + serialized = SiteSerializer(result_page, many=True, context={'request': request}) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response -def delete_site(request, id): + +def get_site(request: object=None, id: str=None) -> object: + """ + Get single `Site` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='site', action='get', + id=id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get site if checks passed + site = Site.objects.get(id=id) + + # serialize and return + serialized = SiteSerializer(site, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_site(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Site` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object, - try: - site = Site.objects.get(id=id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + Returns: + HTTP Response object + """ - if site.account != account: - data = {'reason': 'delete a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='site', + action='delete', id=id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get site if checks passed + site = Site.objects.get(id=id) # remove s3 objects - delete_site_s3_bg.delay(site_id=id) + delete_site_s3_bg.apply_async(kwargs={'site_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # remove any associated tasks + delete_tasks_and_schedules(resource_id=str(site.id), scope='site', account=account) + + # remove any site associated Issues + Issue.objects.filter(affected__icontains=str(id)).delete() + + # remove any page associated Issues + for page in Page.objects.filter(site=site): + Issue.objects.filter(affected__icontains=str(page.id)).delete() # remove site site.delete() - data = {'message': 'Site has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # decrememt resouce in account + decrement_resource(account=account, resource='sites') + + # update account if enterprise or cloud + if account.type == 'enterprise' or account.type == 'cloud': + account.usage['sites_allowed'] -= 1 + account.usage['schedules_allowed'] -= 1 + account.save() + + # update billing for enterprise + if account.type == 'enterprise': + update_sub_price.apply_async(kwargs={'account_id': str(account.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # returning response + data = {'message': 'site deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + -def delete_many_sites(request): + +def delete_many_sites(request: object=None) -> object: + """ + Deletes one or more `Sites` associated + with the passed "request.ids" + + Args: + 'request' : object, + + Returns: + HTTP Response object + """ + + # get request data ids = request.data.get('ids') + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account + # check for ids if ids is not None: + + # setting defaults count = len(ids) num_succeeded = 0 succeeded = [] @@ -223,30 +851,42 @@ def delete_many_sites(request): user = request.user this_status = True + # loop through passed ids for id in ids: + + # trying to delete site try: - site = Site.objects.get(id=id) - if site.account == account: - delete_site_s3_bg.delay(site_id=id) - site.delete() + # delete site and associated resources + data = delete_site(id=id, user=user) + if data.get('reason'): + raise Exception(data['reason']) + + # add to success attempts num_succeeded += 1 succeeded.append(str(id)) - except: + + except Exception as e: + print(e) + # add to failed attempts num_failed += 1 failed.append(str(id)) this_status = False + # format response data = { - 'status': this_status, + 'success': this_status, 'num_succeeded': num_succeeded, 'succeeded': succeeded, 'num_failed': num_failed, 'failed': failed, } + + # returning response record_api_call(request, data, '200') response = Response(data, status=status.HTTP_200_OK) return response + # returning error data = { 'reason': 'you must provide an array of id\'s' } @@ -256,320 +896,527 @@ def delete_many_sites(request): -def create_test(request, delay=False): - # get data from request - configs = request.data.get('configs', None) - pre_scan_id = request.data.get('pre_scan', None) - post_scan_id = request.data.get('post_scan', None) - index = request.data.get('index', None) - test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - tags = request.data.get('tags', None) - pre_scan = None - post_scan = None - site_id = request.data.get('site_id') - user = request.user - account = Member.objects.get(user=user).account - - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) - - site = Site.objects.get(id=site_id, ) - if site.account != account: - data = {'reason': 'create a Test of a Site you do not own'} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - if len(test_type) == 0: - test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] - - if not configs: - configs = { - 'window_size': '1920,1080', - 'interval': 5, - 'driver': 'selenium', - 'device': 'desktop', - 'mask_ids': None, - 'min_wait_time': 10, - 'max_wait_time': 60, - 'timeout': 300, - 'disable_animations': False - } +def get_sites_zapier(request: object=None) -> object: + """ + Get all `Sites` associated with user's Account. - if not Scan.objects.filter(site=site).exists(): - data = {'reason': 'Site not yet onboarded'} - record_api_call(request, data, '400') - return Response(data, status=status.HTTP_400_BAD_REQUEST) + Args: + 'request': object - if pre_scan_id: - try: - pre_scan = Scan.objects.get(id=pre_scan_id) - except: - data = {'reason': 'cannot find a Scan with that id - pre_scan '} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - if post_scan_id: - try: - post_scan = Scan.objects.get(id=post_scan_id) - except: - data = {'reason': 'cannot find a Scan with that id - post_scan '} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - # grabbing most recent Scan - if pre_scan_id is None: - pre_scan = Scan.objects.filter(site=site).order_by('-time_created')[0] + Returns: + HTTP Response object + """ - if pre_scan: - if pre_scan.time_completed == None: - data = {'reason': 'pre_scan still running'} - record_api_call(request, data, '400') - return Response(data, status=status.HTTP_400_BAD_REQUEST) + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + sites = None - if post_scan: - if post_scan.time_completed == None: - data = {'reason': 'post_scan still running'} - record_api_call(request, data, '400') - return Response(data, status=status.HTTP_400_BAD_REQUEST) - + # deciding on scope + resource = 'site' - # creating test object - test = Test.objects.create( - site=site, - type=test_type, - tags=tags, + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, action='get', ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) - - if delay == True: - create_test_bg.delay( - test_id=test.id, - configs=configs, - type=test_type, - index=index, - pre_scan=pre_scan_id, - post_scan=post_scan_id, - tags=tags, - ) - data = { - 'status': True, - 'message': 'test is being created in the background', - 'id': str(test.id), - } - record_api_call(request, data, '201') - return Response(data, status=status.HTTP_201_CREATED) - - else: - if not pre_scan and not post_scan: - new_scan = S(site=site, configs=configs, type=test_type) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan - - if not post_scan and pre_scan: - post_scan = S(site=site, scan=pre_scan, configs=configs, type=test_type).second_scan() - - # updating parired scans - pre_scan.paired_scan = post_scan - post_scan.paried_scan = pre_scan - pre_scan.save() - post_scan.save() - - # updating test object - test.type = test_type - test.type = test_type - test.pre_scan = pre_scan - test.post_scan = post_scan - test.save() - - # running tester - updated_test = T(test=test).run_test(index=index) + # get all account assocoiated sites + if sites is None: + sites = Site.objects.filter( + account=account, + ).order_by('-time_created') - serializer_context = {'request': request,} - serialized = TestSerializer(updated_test, context=serializer_context) - data = serialized.data - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) - return response + # filter out all non permissioned sites + if len(member.permissions.get('sites', [])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + sites = sites.filter(id__in=id_list).order_by('-time_created') + # build response data + data = [] + for site in sites: + data.append({ + 'id' : str(site.id), + 'site_url' : str(site.site_url), + 'time_created' : str(site.time_created), + 'tags' : site.tags, + 'info' : site.info, + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response +### ------ Begin Page Services ------ ### -def get_tests(request): - user = request.user - account = Member.objects.get(user=user).account - test_id = request.query_params.get('test_id') - site_id = request.query_params.get('site_id') - time_begin = request.query_params.get('time_begin') - time_end = request.query_params.get('time_end') - lean = request.query_params.get('lean') - - if test_id != None: - try: - test = Test.objects.get(id=test_id) - except: - data = {'reason': 'cannot find a Test with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) +def create_or_update_page(request: object=None) -> object: + """ + Creates one or more pages. - if test.site.account != account: - data = {'reason': 'retrieve Tests of a Site you do not own'} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = TestSerializer(test, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + Args: + 'requests': object + + Returns: + HTTP Response object + """ + # getting request data + site_id = request.data.get('site_id') + page_id = request.data.get('page_id') + page_url = request.data.get('page_url') + page_urls = request.data.get('page_urls') + tags = request.data.get('tags', None) + configs = request.data.get('configs', None) + no_scan = request.data.get('no_scan', False) - try: - site = Site.objects.get(id=site_id) - except: - if site_id != None: - data = {'reason': 'cannot find a site with that id',} - this_status = status.HTTP_404_NOT_FOUND - status_code = '404' - else: - data = {'reason': 'you did not provide the site_id'} - this_status = status.HTTP_400_BAD_REQUEST - status_code = '400' - record_api_call(request, data, status_code) - return Response(data, status=this_status) - - if site.account != account: - data = {'reason': 'retrieve Tests of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # retrieving user, account, & site + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # creating many pages if page_urls was passed + if page_urls is not None: + data = create_many_pages(request=request, http_response=False) + _status = status.HTTP_201_CREATED + if data.get('reason') is not None: + _status = status.HTTP_402_PAYMENT_REQUIRED + response = Response(data, status=_status) + return response - if time_begin == None and site != None and time_end != None: - tests = Test.objects.filter(site=site).filter(time_completed__lte=time_end).order_by('-time_created') - elif time_end == None and site != None and time_begin != None: - tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).order_by('-time_created') - elif time_end != None and time_begin != None and site != None: - tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created') - elif time_end == None and time_begin == None and Site != None: - tests = Test.objects.filter(site=site).order_by('-time_created') - else: - data = {'reason': 'you did not provide the right params',} + # validating page_url + if (page_url is None or page_url == '') and not page_id: + data = {'reason': 'the page_url cannot be empty'} record_api_call(request, data, '400') return Response(data, status=status.HTTP_400_BAD_REQUEST) + if page_url: + if page_url.endswith('/'): + page_url = page_url.rstrip('/') + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='page', action='add' if site_id else 'update', + id=site_id if site_id else page_id, + id_type='site' if site_id else 'page', url=page_url, + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # update page if page_id passed + if page_id: - paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(tests, request) - serializer_context = {'request': request,} - serialized = TestSerializer(result_page, many=True, context=serializer_context) - if lean is not None: - serialized = SmallTestSerializer(result_page, many=True, context=serializer_context) - - response = paginator.get_paginated_response(serialized.data) - record_api_call(request, response.data, '200') + # get page & update data + page = Page.objects.get(id=page_id) + if tags is not None: + page.tags = tags + page.save() + + # serialize response and return + serialized = PageSerializer(page, context={'request': request}) + record_api_call(request, serialized.data, '200') + response = Response(serialized.data, status=status.HTTP_200_OK) + return response + + # get site + site = Site.objects.get(id=site_id) + + # adding page if checks passed + page = Page.objects.create( + site=site, + page_url=page_url, + user=user, + tags=tags, + account=account + ) + + # deciding on scan + if no_scan == False: + # create initial scan + scan = Scan.objects.create( + site=site, + page=page, + type=settings.TYPES, + configs=configs + ) + page.info["latest_scan"]["id"] = str(scan.id) + page.info["latest_scan"]["time_created"] = str(scan.time_created) + page.save() + + # running scan in background + scan_page_bg.apply_async(kwargs={'scan_id': str(scan.id), '_queue': ON_DEMAND_QUEUE}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # serialize response and return + serialized = PageSerializer(page, context={'request': request}) + data = serialized.data + record_api_call(request, serialized.data, '201') + response = Response(serialized.data, status=status.HTTP_201_CREATED) return response +def create_many_pages(request: object, http_response: bool=True) -> object: + """ + Bulk creates `Pages` for each url passed in "page_urls" + + Args: + 'request' : object, + 'http_response' : bool + + Returns: dict or HTTP Response object + """ -def get_test_lean(request, id): + # get request data + site_id = request.data.get('site_id') + page_urls = request.data.get('page_urls') + tags = request.data.get('tags', None) + configs = request.data.get('configs', None) + no_scan = request.data.get('no_scan', False) + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account - try: - test = Test.objects.get(id=id) - except: - data = {'reason': 'cannot find a Test with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # updating configs if None: + configs = account.configs if configs == None else configs - if test.site.account != account: - data = {'reason': 'retrieve Tests of a Site you do not own'} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # get site and current pages + site = Site.objects.get(id=site_id) + pages = Page.objects.filter(site=site) - # get images_delta if exists - try: - images_delta = {"average_score": test.images_delta.get('average_score')} - except: - images_delta = None + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='page', action='add', + id=site_id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + if http_response: + return Response(data, status=check_data['status']) + return data + + # pre check for max_pages + if (pages.count() + len(page_urls)) > account.usage['pages_allowed']: + print('max pages reached') + data = {'reason': 'max pages reached'} + record_api_call(request, data, '402') + if http_response: + return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + return data + + # setting defaults + count = len(page_urls) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # looping through each "page_url" + for url in page_urls: + + # clean url + if url.endswith('/'): + url = url.rstrip('/') + + # check for duplicates + if not Page.objects.filter(page_url=url, user=user).exists(): + + # adding pages + page = Page.objects.create( + site=site, + page_url=url, + user=user, + tags=tags, + account=account + ) - # get lighthouse_delta if exists - try: - lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')} - except: - lighthouse_delta = None + # deciding on scan + if no_scan == False: - # get lighthouse_delta if exists - try: - yellowlab_delta = {"scores": test.yellowlab_delta['scores']} - except: - yellowlab_delta = None + # create initial scan + scan = Scan.objects.create( + site=site, + page=page, + type=settings.TYPES, + configs=configs + ) + + # update page with new scan data + page.info["latest_scan"]["id"] = str(scan.id) + page.info["latest_scan"]["time_created"] = str(scan.time_created) + page.save() + + # run scanner + scan_page_bg.apply_async( + kwargs={'scan_id': str(scan.id), '_queue': ON_DEMAND_QUEUE}, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # update info + succeeded.append(url) + num_succeeded = num_succeeded + 1 + + else: + # update info + this_status = False + failed.append(url) + num_failed = num_failed + 1 + # formatting response data = { - "id": str(test.id), - "site": str(test.site.id), - "tags": test.tags, - "type": test.type, - "time_created": str(test.time_created), - "time_completed": str(test.time_completed), - "pre_scan": str(test.pre_scan.id), - "post_scan": str(test.post_scan.id), - "score": test.score, - "lighthouse_delta": lighthouse_delta, - "yellowlab_delta": yellowlab_delta, - "images_delta": images_delta, + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, } - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # record successful API call + record_api_call(request, data, '201') + + # decide on response type + if http_response: + print('requested http response') + # returning HTTP Response + response = Response(data, status=status.HTTP_201_CREATED) + return response + + # return dict response + print('requested data response') + return data + + + +def get_pages(request: object=None) -> object: + """ + Get one or more `Pages` from either + "page_id" or "site_id" + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + site_id = request.query_params.get('site_id') + page_id = request.query_params.get('page_id') + sort = request.query_params.getlist('sort') + # get user and account + user = request.user + member = Member.objects.get(user=user) + # check for params + if page_id is None and site_id is None: + data = {'reason': 'need site_id or page_id'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='page', action='get', + id=(site_id if site_id else page_id), + id_type=('site' if site_id else 'page') + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # page-specific sorting dict + sorting_items = { + # accend + 'page': 'page_url', + 'time_created': 'time_created', + 'scan': 'info__latest_scan__score', + 'scan_completed': 'info__latest_scan__time_completed', + 'test': 'info__latest_test__score', + # decend + '-page': '-page_url', + '-time_created': '-time_created', + '-scan': '-info__latest_scan__score', + '-scan_completed': '-info__latest_scan__time_completed', + '-test': '-info__latest_test__score', + } + # transform sort param + _sort = [] + for s in sort: + _sort.extend(s.split(',')) -def delete_test(request, id): - try: - test = Test.objects.get(id=id) - except: - data = {'reason': 'cannot find a Test with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # build order_by list + ordering = [sorting_items[s] for s in _sort if s in sorting_items] + if not ordering: + ordering = ['-time_created'] + + # getting single page + if page_id != None: - site = test.site - user = request.user - account = Member.objects.get(user=user).account + # get page + page = Page.objects.get(id=page_id) - if site.account != account: - data = {'reason': 'delete Tests of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # serialize and return + serialized = PageSerializer(page, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) - test.delete() + # get site and assocaited pages + site = Site.objects.get(id=site_id) + pages = Page.objects.filter(site=site).order_by(*ordering) - data = {'message': 'Test has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(pages, request) + serialized = PageSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') return response -def delete_many_tests(request): + +def get_page(request: object=None, id: str=None) -> object: + """ + Get single `Page` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='page', action='get', + id=id, id_type='page' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get page if checks passed + page = Page.objects.get(id=id) + + # serialize and return + serialized = PageSerializer(page, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_page(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Page` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='page', + action='delete', id=id, id_type='page' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + print(data) + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get page by id + page = Page.objects.get(id=id) + + # remove s3 objects + delete_page_s3_bg.apply_async(kwargs={'page_id': str(id), 'site_id': str(page.site.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # remove any schedules and associated tasks + delete_tasks_and_schedules(resource_id=str(page.id), scope='page', account=account) + + # remove any associated Issues + Issue.objects.filter(affected__icontains=str(id)).delete() + + # remove page + page.delete() + + # format and return + data = {'message': 'Page deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def delete_many_pages(request: object=None) -> object: + """ + Deletes one or more `Pages` associated + with the passed "request.ids" + + Args: + 'request' : object, + + Returns: + HTTP Response object + """ + + # get request data ids = request.data.get('ids') + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account + # check for ids if ids is not None: + + # setting defaults count = len(ids) num_succeeded = 0 succeeded = [] @@ -578,29 +1425,41 @@ def delete_many_tests(request): user = request.user this_status = True + # loop through passed ids for id in ids: + + # trying to delete page try: - test = Test.objects.get(id=id) - if test.site.account == account: - test.delete() + # delete page and all assocaited resourses + data = delete_page(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts num_succeeded += 1 succeeded.append(str(id)) - except: + except Exception as e: + # add to failed attempts + print(e) num_failed += 1 failed.append(str(id)) this_status = False + # format data data = { - 'status': this_status, + 'success': this_status, 'num_succeeded': num_succeeded, 'succeeded': succeeded, 'num_failed': num_failed, 'failed': failed, } + + # returning response record_api_call(request, data, '200') response = Response(data, status=status.HTTP_200_OK) return response + # returning error data = { 'reason': 'you must provide an array of id\'s' } @@ -611,156 +1470,424 @@ def delete_many_tests(request): +def get_pages_zapier(request: object=None) -> object: + """ + Get all `Pages` associated with user's Account. -def create_scan(request, delay=False): + Args: + 'request': object - user = request.user - account = Member.objects.get(user=user).account - site_id = request.data.get('site_id', None) - configs = request.data.get('configs', None) - types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - tags = request.data.get('tags', None) + Returns: + HTTP Response object + """ - if len(types) == 0: - types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + # get request data + member = Member.objects.get(user=request.user) + account = member.account + site_id = request.query_params.get('site_id') + pages = None - try: - site = Site.objects.get(id=site_id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # deciding on scope + resource = 'page' - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, action='get', + id=site_id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all site associated pages + if site_id: + pages = Page.objects.filter( + account=account, + site__id=site_id, + ).order_by('-time_created') + + # get all account assocoiated pages + if pages is None: + pages = Page.objects.filter( + account=account, + ).order_by('-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + pages = pages.filter(site__id__in=id_list) + + # build response data + data = [] + + for page in pages: + data.append({ + 'id' : str(page.id), + 'page_url' : str(page.page_url), + 'site' : str(page.site.id), + 'site_url' : str(page.site.site_url), + 'time_created' : str(page.time_created), + 'tags' : page.tags, + 'info' : page.info, + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin Scan Services ------ ### + + + + +def create_scan(request: object=None, **kwargs) -> object: + """ + Create one or more `Scans` depanding on + `Page` or `Site` scope + + Args: + 'request': object, - if site.account != account: - data = {'reason': 'create a Scan of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + Returns: + dict or HTTP Response object + """ + + # check location + location_data = check_location(request, None) + if location_data['routed']: + return location_data['response'] + + # get request data + if request is not None: + site_id = request.data.get('site_id', '') + page_id = request.data.get('page_id', '') + configs = request.data.get('configs', None) + types = request.data.get('type', settings.TYPES) + tags = request.data.get('tags') + user = request.user + + # getting kwargs data + if request is None: + site_id = kwargs.get('site_id', '') + page_id = kwargs.get('page_id', '') + configs = kwargs.get('configs', None) + types = kwargs.get('type', settings.TYPES) + tags = kwargs.get('tags') + user_id = kwargs.get('user_id') + user = User.objects.get(id=user_id) + # getting account + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # checking args + site_id = '' if site_id is None else site_id + page_id = '' if page_id is None else page_id + site_id = site_id if len(str(site_id)) > 0 else None + page_id = page_id if len(str(page_id)) > 0 else None + id = site_id if site_id else page_id + id_type = 'site' if site_id else 'page' + + # verifying types + if len(types) == 0: + types = settings.TYPES - if not configs: - configs = { - 'window_size': '1920,1080', - 'interval': 5, - 'driver': 'selenium', - 'device': 'desktop', - 'mask_ids': None, - 'min_wait_time': 10, - 'max_wait_time': 60, - 'timeout': 300, - 'disable_animations': False + # check account and resource for site or page + check_data = check_permissions_and_usage( + member=member, resource='scan', action='add', + id=id, id_type=id_type + ) + if not check_data['allowed']: + data = { + 'reason': check_data['error'], + 'success': False, + 'code': check_data['code'], + 'status': check_data['status'] } + if request is not None: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data - # creating scan obj - created_scan = Scan.objects.create( - site=site, - tags=tags, - type=types, - configs=configs, - ) + # get site or page + if site_id is not None: + site = Site.objects.get(id=site_id) + if page_id is not None: + page = Page.objects.get(id=page_id) + + # setting pages to loop through + if site_id is not None and page_id is None: + pages = Page.objects.filter(site=site) + if site_id is None and page_id is not None: + pages = [page,] + + # setting default + created_scans = [] + queue = ON_DEMAND_QUEUE + + # looping through each page + for p in pages: + + # check for account usage + check_data = check_permissions_and_usage( + member=member, resource='scan', action='add', + ) + if not check_data['allowed']: + data = { + 'reason': check_data['error'], + 'success': False, + 'code': check_data['code'], + 'status': check_data['status'] + } + if request is not None: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # update usage and meter resource + check_and_increment_resource(account.id, 'scans') + + # creating scan obj + created_scan = Scan.objects.create( + site=p.site, + page=p, + tags=tags, + type=types, + configs=configs + ) - if delay == True: + # adding system data + add_scan_system_data( + scan=created_scan, + kwargs={ + 'scan_id': str(created_scan.id), + 'alert_id': None, + 'flowrun_id': None, + 'node_index': None, + '_queue': queue, + } + ) - # running scans in selenium mode + # adding scan to array + created_scans.append(str(created_scan.id)) + message = 'Scans are being created in the background' + + # setting format for timestamp + f = '%Y-%m-%d %H:%M:%S.%f' + timestamp = datetime.today().strftime(f) + + # updating latest_scan info for page + p.info['latest_scan']['id'] = str(created_scan.id) + p.info['latest_scan']['time_created'] = timestamp + p.info['latest_scan']['time_completed'] = None + p.info['latest_scan']['score'] = None + p.info['latest_scan']['score'] = None + p.save() + + # updating latest_scan info for site + p.site.info['latest_scan']['id'] = str(created_scan.id) + p.site.info['latest_scan']['time_created'] = timestamp + p.site.info['latest_scan']['time_completed'] = None + p.site.save() + + # running scans components in parallel if 'html' in types or 'logs' in types or 'full' in types: - print('running html & logs') - run_html_and_logs_bg.delay(scan_id=created_scan.id) + task_id = f'lock:html_and_logs_bg_{created_scan.id}' + run_html_and_logs_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id) + if 'lighthouse' in types or 'full' in types: - print('running lighthouse') - run_lighthouse_bg.delay(scan_id=created_scan.id) + task_id = f'lock:lighthouse_bg_{created_scan.id}' + run_lighthouse_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id) + if 'yellowlab' in types or 'full' in types: - print('running yellowlab') - run_yellowlab_bg.delay(scan_id=created_scan.id) + task_id = f'lock:yellowlab_bg_{created_scan.id}' + run_yellowlab_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id) + if 'vrt' in types or 'full' in types: - print('running vrt') - run_vrt_bg.delay(scan_id=created_scan.id) - - - data = { - 'status': True, - 'message': 'scan is being created in the background', - 'id': str(created_scan.id), - } + task_id = f'lock:vrt_bg_{created_scan.id}' + run_vrt_bg.apply_async(kwargs={'scan_id': str(created_scan.id), '_queue': queue}, queue=queue, routing_key=queue, task_id=task_id) + + # returning dynaminc response + data = { + 'success': True, + 'message': message, + 'ids': created_scans, + } + if request is not None: record_api_call(request, data, '201') return Response(data, status=status.HTTP_201_CREATED) - else: - updated_scan = S(scan=created_scan, configs=configs).first_scan() - serializer_context = {'request': request,} - serialized = ScanSerializer(updated_scan, context=serializer_context) - data = serialized.data - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) - return response - + return data -def get_scans(request): +def create_many_scans(request: object=None) -> object: + """ + Bulk creates `Scans` for each requested `Page`. + Either scoped for many `Pages` or many `Sites`. + Args: + 'request' : object, + + Returns: + HTTP Response object + """ + + # check location + location_data = check_location(request, None) + if location_data['routed']: + return location_data['response'] + + # get request data + site_ids = request.data.get('site_ids') + page_ids = request.data.get('page_ids') + configs = request.data.get('configs', None) + types = request.data.get('type', settings.TYPES) + tags = request.data.get('tags') user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # setting defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # scoped for sites + if site_ids: + for id in site_ids: + data = { + 'site_id': str(id), + 'configs': configs, + 'type': types, + 'tags': tags, + 'user_id': str(user.id) + } + try: + # create scan + res = create_scan(**data) + if res['success']: + num_succeeded += 1 + succeeded.append(str(id)) + else: + num_failed += 1 + this_status = False + failed.append(str(id)) + print(res['reason']) + except Exception as e: + print(e) + if str(id) not in failed: + num_failed += 1 + this_status = False + failed.append(str(id)) + + # scoped for pages + if page_ids: + for id in page_ids: + data = { + 'page_id': str(id), + 'configs': configs, + 'type': types, + 'tags': tags, + 'user_id': str(user.id) + } + try: + # create scan + res = create_scan(**data) + if res['success']: + num_succeeded += 1 + succeeded.append(str(id)) + else: + num_failed += 1 + this_status = False + failed.append(str(id)) + print(res['reason']) + except Exception as e: + print(e) + if str(id) not in failed: + num_failed += 1 + this_status = False + failed.append(str(id)) + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '201') + return Response(data, status=status.HTTP_201_CREATED) + + + + +def get_scans(request: object=None) -> object: + """ + Get one or more `Scans`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data scan_id = request.query_params.get('scan_id') - site_id = request.query_params.get('site_id') - time_begin = request.query_params.get('time_begin') - time_end = request.query_params.get('time_end') + page_id = request.query_params.get('page_id') lean = request.query_params.get('lean') + user = request.user + member = Member.objects.get(user=user) + + # deciding on scope + id = page_id if page_id else scan_id + id_type = 'page' if page_id else 'scan' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='scan', + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + # get single scan if scan_id != None: - try: - scan = Scan.objects.get(id=scan_id) - except: - data = {'reason': 'cannot find a Scan with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - if scan.site.account != account: - data = {'reason': 'retrieve Scans of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # get scan + scan = Scan.objects.get(id=scan_id) - serializer_context = {'request': request,} - serialized = ScanSerializer(scan, context=serializer_context) + # serialize and return + serialized = ScanSerializer(scan, context={'request': request}) data = serialized.data record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) - - try: - site = Site.objects.get(id=site_id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - - if site.account != account: - data = {'reason': 'retrieve Scans of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - - if time_begin == None and site != None and time_end != None: - scans = Scan.objects.filter(site=site).filter(time_created__lte=time_end).order_by('-time_created') - elif time_end == None and site != None and time_begin != None: - scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).order_by('-time_created') - elif time_end == None and time_begin == None and site != None: - scans = Scan.objects.filter(site=site).order_by('-time_created') - elif time_end != None and time_begin != None and site != None: - scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created') - + # get page scoped scans + page = Page.objects.get(id=page_id) + scans = Scan.objects.filter(page=page).order_by('-time_created') + # serialize and return paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(scans, request) - serializer_context = {'request': request,} - serialized = ScanSerializer(result_page, many=True, context=serializer_context) - if lean is not None: - serialized = SmallScanSerializer(result_page, many=True, context=serializer_context) + serialized = ScanSerializer(result_page, many=True, context={'request': request}) + if str(lean).lower() == 'true': + serialized = SmallScanSerializer(result_page, many=True, context={'request': request}) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response @@ -768,34 +1895,79 @@ def get_scans(request): -def get_scan_lean(request, id): +def get_scan(request: object=None, id: str=None) -> object: + """ + Get single `Scan` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) - try: - scan = Scan.objects.get(id=id) - except: - data = {'reason': 'cannot find a Scan with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='scan', action='get', + id=id, id_type='scan' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - if scan.site.account != account: - data = {'reason': 'retrieve Scans of a Site you do not own'} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # get scan if checks passed + scan = Scan.objects.get(id=id) + + # serialize and return + serialized = ScanSerializer(scan, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) - # get lighthouse scores if exists - try: - lighthouse = {"scores": scan.lighthouse.get('scores')} - except: - lighthouse = None + + + +def get_scan_lean(request: object=None, id: str=None) -> object: + """ + Get a single `Scan` and only return scores & timestamps + + Args: + 'request' : object, + 'id' : str - # get yellowlab scores if exists - try: - yellowlab = {"scores": scan.yellowlab.get('scores')} - except: - yellowlab = None + Returns: + HTTP Response object + """ + # get user and account + user = request.user + member = Member.objects.get(user=user) + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='scan', action='get', + id=id, id_type='scan' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get scan if checks passed + scan = Scan.objects.get(id=id) + + # get lighthouse scores if exists + lighthouse = {"scores": scan.lighthouse.get('scores')} + + # get yellowlab scores if exists + yellowlab = {"scores": scan.yellowlab.get('scores')} + + # format data data = { "id": str(scan.id), "site": str(scan.site.id), @@ -807,6 +1979,7 @@ def get_scan_lean(request, id): "yellowlab": yellowlab, } + # return response record_api_call(request, data, '200') response = Response(data, status=status.HTTP_200_OK) return response @@ -814,39 +1987,89 @@ def get_scan_lean(request, id): -def delete_scan(request, id): - try: - scan = Scan.objects.get(id=id) - except Exception as e: - data = {'reason': 'cannot find a Scan with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - site = scan.site - user = request.user - account = Member.objects.get(user=user).account +def delete_scan(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Scan` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'account' : object, + 'user' : object + + Returns: + HTTP Response object + """ + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account - if site.account != account: - data = {'reason': 'delete Scans of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='scan', action='delete', + id=id, id_type='scan' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get scan if checks passes + scan = Scan.objects.get(id=id) + + # remove s3 objects + delete_scan_s3_bg.apply_async(kwargs={'scan_id': str(scan.id), 'site_id': str(scan.site.id), 'page_id': str(scan.page.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + # delete scan + page_id = str(scan.page.id) scan.delete() - data = {'message': 'Scan has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # update page and site + update_site_and_page_info.apply_async( + kwargs={'resource': 'scan', 'page_id': str(page_id)}, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # return response + data = {'message': 'Scan deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data -def delete_many_scans(request): + +def delete_many_scans(request: object=None) -> object: + """ + Deletes one or more `Scans` associated + with the passed "request.ids" + + Args: + 'request' : object, + + Returns: + HTTP Response object + """ + + # get request data ids = request.data.get('ids') + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + # check for ids if ids is not None: + + # setting defaults count = len(ids) num_succeeded = 0 succeeded = [] @@ -855,30 +2078,41 @@ def delete_many_scans(request): user = request.user this_status = True + # loop through passed ids for id in ids: + + # trying to delete scan try: - scan = Scan.objects.get(id=id) - if scan.site.account == account: - scan.delete() + # delete scan and all assocaited resourses + data = delete_scan(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts num_succeeded += 1 succeeded.append(str(id)) - except: + except Exception as e: + # add to failed attempts + print(e) num_failed += 1 failed.append(str(id)) this_status = False + # format data data = { - 'status': this_status, + 'success': this_status, 'num_succeeded': num_succeeded, 'succeeded': succeeded, 'num_failed': num_failed, 'failed': failed, } + # returning response record_api_call(request, data, '200') response = Response(data, status=status.HTTP_200_OK) return response + # return error data = { 'reason': 'you must provide an array of id\'s' } @@ -889,257 +2123,529 @@ def delete_many_scans(request): +def get_scans_zapier(request: object=None) -> object: + """ + Get all `Scans` associated with user's Account. -def create_or_update_schedule(request): - user = request.user - account = Member.objects.get(user=user).account + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + member = Member.objects.get(user=request.user) + account = member.account + page_id = request.query_params.get('page_id') + site_id = request.query_params.get('site_id') + id = page_id if page_id else site_id + id_type = 'page' if page_id else 'site' + scans = None + + # deciding on scope + resource = 'scan' - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all page associated scans + if page_id: + scans = Scan.objects.filter( + page__account=account, + page__id=page_id, + ).exclude( + time_completed=None, + ).order_by('-time_created') + + # get all site associated scans + if site_id: + scans = Scan.objects.filter( + site__account=account, + site__id=site_id, + ).exclude( + time_completed=None, + ).order_by('-time_created') + + # get all account assocoiated scans + if scans is None: + scans = Scan.objects.filter( + site__account=account, + ).exclude( + time_completed=None, + ).order_by('-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + scans = scans.filter(site__id__in=id_list).order_by('-time_created') + + # build response data + data = [] + + for scan in scans: + data.append({ + 'id' : str(scan.id), + 'page' : str(scan.page.id), + 'site' : str(scan.site.id), + 'time_created' : str(scan.time_created), + 'time_completed' : str(scan.time_completed), + 'type' : scan.type, + 'html' : scan.html, + 'logs' : scan.logs, + 'images' : scan.images, + 'lighthouse' : scan.lighthouse, + 'yellowlab' : scan.yellowlab, + 'configs' : scan.configs, + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response - try: - site = Site.objects.get(id=request.data.get('site_id')) - if site.account != account and site.account != None: - data = {'reason': 'create a Schedule of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - except: - site = None - try: - schedule = Schedule.objects.get(id=request.data.get('schedule_id')) - if schedule.account != account and schedule.account != None: - data = {'reason': 'update a Schedule you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - except: - schedule = None - - - schedule_status = request.data.get('status', None) - begin_date_raw = request.data.get('begin_date', None) - time = request.data.get('time', None) - timezone = request.data.get('timezone', None) - freq = request.data.get('frequency', None) - task_type = request.data.get('task_type', None) - test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']) - configs = request.data.get('configs', None) - schedule_id = request.data.get('schedule_id', None) - case_id = request.data.get('case_id', None) - updates = request.data.get('updates', None) - - - if configs is None: - configs = { - 'window_size': '1920,1080', - 'driver': 'selenium', - 'device': 'desktop', - 'mask_ids': None, - 'interval': 5, - 'min_wait_time': 10, - 'max_wait_time': 30, - 'timeout': 300, - 'disable_animations': False - } - if schedule_status != None and schedule != None: - task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - if task.enabled == True: - task.enabled = False - schedule.status = 'Paused' - else: - task.enabled = True - schedule.status = 'Active' - task.save() - schedule.save() - # retriving object again to avoid cacheing issues - schedule_new = Schedule.objects.get(id=request.data.get('schedule_id')) - - # if not status change, updating data - else: - if Automation.objects.filter(schedule=schedule).exists(): - automation = Automation.objects.filter(schedule=schedule)[0] - auto_id = str(automation.id) - else: - auto_id = None - if task_type == 'test': - task = 'api.tasks.create_test_bg' - arguments = { - 'site_id': str(site.id), - 'configs': configs, - 'type': test_type, - 'automation_id': auto_id - } - if task_type == 'scan': - task = 'api.tasks.create_scan_bg' - arguments = { - 'site_id': str(site.id), - 'configs': configs, - 'type': scan_type, - 'automation_id': auto_id - } +### ------ Begin Test Services ------ ### - if task_type == 'report': - task = 'api.tasks.create_report_bg' - arguments = { - 'site_id': str(site.id), - 'automation_id': auto_id - } - if task_type == 'testcase': - task = 'api.tasks.create_testcase_bg' - arguments = { - 'site_id': str(site.id), - 'case_id': str(case_id), - 'updates': updates, - 'configs': configs, - 'automation_id': auto_id, - } - format_str = '%m/%d/%Y' - - try: - begin_date = datetime.strptime(begin_date_raw, format_str) - except: - begin_date = datetime.now() +def create_test(request: object=None, **kwargs) -> object: + """ + Create one or more `Tests` depanding on + `Page` or `Site` scope - num_day_of_week = begin_date.weekday() - day = begin_date.strftime("%d") - minute = time[3:5] - hour = time[0:2] + Args: + 'request': object, + 'delay': bool + + Returns: + dict or HTTP Response object + """ - if freq == 'daily': - day_of_week = '*' - day_of_month = '*' - elif freq == 'weekly': - day_of_week = num_day_of_week - day_of_month = '*' - elif freq == 'monthly': - day_of_week = '*' - day_of_month = day + # check location + location_data = check_location(request, None) + if location_data['routed']: + return location_data['response'] + # get data from request + if request is not None: + configs = request.data.get('configs', None) + threshold = request.data.get('threshold', settings.TEST_THRESHOLD) + pre_scan_id = request.data.get('pre_scan') + post_scan_id = request.data.get('post_scan') + index = request.data.get('index') + test_type = request.data.get('type', settings.TYPES) + tags = request.data.get('tags') + pre_scan = None + post_scan = None + site_id = request.data.get('site_id', '') + page_id = request.data.get('page_id', '') + user = request.user - task_name = str(task_type) + '_' + str(site.site_url) + '_' + str(freq) + '_@' + str(time) + # get data from kwargs + if request is None: + configs = kwargs.get('configs', None) + threshold = kwargs.get('threshold', settings.TEST_THRESHOLD) + pre_scan_id = kwargs.get('pre_scan') + post_scan_id = kwargs.get('post_scan') + index = kwargs.get('index') + test_type = kwargs.get('type', settings.TYPES) + tags = kwargs.get('tags') + pre_scan = None + post_scan = None + site_id = kwargs.get('site_id', '') + page_id = kwargs.get('page_id', '') + user_id = kwargs.get('user_id') + user = User.objects.get(id=user_id) + + # get account + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # verifying test_type + if len(test_type) == 0: + test_type = settings.TYPES + + # checking args + site_id = '' if site_id is None else site_id + page_id = '' if page_id is None else page_id + site_id = site_id if len(str(site_id)) > 0 else None + page_id = page_id if len(str(page_id)) > 0 else None + id = site_id if site_id else page_id + id_type = 'site' if site_id else 'page' + + # check account and resource for page or site + check_data = check_permissions_and_usage( + member=member, resource='test', action='add', + id=id, id_type=id_type + ) + if not check_data['allowed']: + data = { + 'reason': check_data['error'], + 'success': False, + 'code': check_data['code'], + 'status': check_data['status'] + } + if request is not None: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data - crontab, _ = CrontabSchedule.objects.get_or_create( - timezone=timezone, minute=minute, hour=hour, - day_of_week=day_of_week, day_of_month=day_of_month, - ) + # deciding on scope + if site_id is not None: + site = Site.objects.get(id=site_id) + if page_id is not None: + page = Page.objects.get(id=page_id) - if schedule: - if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists(): - periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id) - periodic_task.update( - crontab=crontab, - name=task_name, task=task, - kwargs=json.dumps(arguments), - ) - periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - else: - periodic_task = PeriodicTask.objects.create( - crontab=crontab, name=task_name, task=task, - kwargs=json.dumps(arguments), - ) + # building pages list + if site_id is not None and page_id is None: + pages = Page.objects.filter(site=site) + if site_id is None and page_id is not None: + pages = [page] - else: - if PeriodicTask.objects.filter(name=task_name).exists(): - data = {'reason': 'Task has already be created',} - record_api_call(request, data, '401') - return Response(data, status=status.HTTP_401_UNAUTHORIZED) - - periodic_task = PeriodicTask.objects.create( - crontab=crontab, name=task_name, task=task, - kwargs=json.dumps(arguments), - ) + # setting default + created_tests = [] - extras = { - "configs": configs, - "test_type": test_type, - "scan_type": scan_type, - "case_id": case_id, - "updates": updates - } + # looping through pages + for p in pages: + + # check for account usage + check_data = check_permissions_and_usage( + member=member, action='add', resource='test' + ) + if not check_data['allowed']: + data = { + 'reason': check_data['error'], + 'success': False, + 'code': check_data['code'], + 'status': check_data['status'] + } + if request is not None: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # checking for scan completion + if not Scan.objects.filter(page=p).exists(): + data = {'reason': 'Page not yet onboarded', 'success': False} + print(data) + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) - if schedule: - schedule_query = Schedule.objects.filter(id=schedule_id) - if schedule_query.exists(): - schedule_query.update( - user=request.user, timezone=timezone, - begin_date=begin_date, time=time, frequency=freq, - task=task, crontab_id=crontab.id, task_type=task_type, - extras=extras, account=account - ) - schedule_new = Schedule.objects.get(id=schedule_id) - else: - schedule_new = Schedule.objects.create( - user=request.user, site=site, task_type=task_type, timezone=timezone, - begin_date=begin_date, time=time, frequency=freq, - task=task, crontab_id=crontab.id, - periodic_task_id=periodic_task.id, - extras=extras, - account=account - ) + # verifying pre_ and post_ scans exists + if pre_scan_id: + try: + pre_scan = Scan.objects.get(id=pre_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False} + print(data) + if request is not None: + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + return data + if post_scan_id: + try: + post_scan = Scan.objects.get(id=post_scan_id) + except: + data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False} + print(data) + if request is not None: + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + return data + + # grabbing most recent Scan with matching window size and types + if pre_scan_id is None: + pre_scan = Scan.objects.filter( + page=p, + configs__window_size=configs.get('window_size'), + ).filter( + reduce(operator.and_, (Q(type__contains=[t]) for t in test_type)) + ).order_by('-time_created').first() + + # verifying pre_ and post_ scans completion + if pre_scan: + if pre_scan.time_completed == None: + data = {'reason': 'pre_scan still running', 'success': False} + print(data) + if request is not None: + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + return data + if post_scan: + if post_scan.time_completed == None: + data = {'reason': 'post_scan still running', 'success': False} + print(data) + if request is not None: + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + return data + + # creating test object + test = Test.objects.create( + site=p.site, + page=p, + type=test_type, + tags=tags, + threshold=float(threshold), + status='working', + ) - serializer_context = {'request': request,} - data = ScheduleSerializer(schedule_new, context=serializer_context).data - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # setting format for timestamp + f = '%Y-%m-%d %H:%M:%S.%f' + timestamp = datetime.today().strftime(f) + + # updating latest_test info for page + p.info['latest_test']['id'] = str(test.id) + p.info['latest_test']['time_created'] = timestamp + p.info['latest_test']['time_completed'] = None + p.info['latest_test']['score'] = None + p.info['latest_test']['status'] = 'working' + p.save() + + # updating latest_test info for site + p.site.info['latest_test']['id'] = str(test.id) + p.site.info['latest_test']['time_created'] = timestamp + p.site.info['latest_test']['time_completed'] = None + p.site.info['latest_test']['score'] = None + p.site.info['latest_test']['status'] = 'working' + p.site.save() + + # add test.id to list + created_tests.append(str(test.id)) + + # update usage and meter resource + check_and_increment_resource(account.id, 'tests') + + # creating test system data + test_system = { + "tasks": [ + { + "kwargs": { + "test_id": str(test.id), + "alert_id": None, + "flowrun_id": None, + "node_index": None + }, + "task_id": f"lock:run_test_{test.id}", + "attempts": 0, + "component": "test", + "task_method": "run_test" + } + ] + } + + # update test + test.system = test_system + test.save() + + # running test in background + create_test_bg.apply_async( + kwargs={ + 'test_id': str(test.id), + 'configs': configs, + 'type': test_type, + 'index': index, + 'pre_scan': pre_scan_id, + 'post_scan': post_scan_id, + 'tags': tags, + 'threshold': float(threshold), + '_queue': ON_DEMAND_QUEUE, + }, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + message = 'Tests are being created in the background' + + # returning dynaminc response + data = { + 'success': True, + 'message': message, + 'ids': created_tests, + } + if request is not None: + record_api_call(request, data, '201') + return Response(data, status=status.HTTP_201_CREATED) + return data -def get_schedules(request): +def create_many_tests(request: object=None) -> object: + """ + Bulk creates `Tests` for each requested `Page`. + Either scoped for many `Pages` or many `Sites`. + + Args: + 'request' : object, + + Returns: + HTTP Response object + """ + + # check location + location_data = check_location(request, None) + if location_data['routed']: + return location_data['response'] + + # get request data + site_ids = request.data.get('site_ids') + page_ids = request.data.get('page_ids') + configs = request.data.get('configs', None) + threshold = request.data.get('threshold', settings.TEST_THRESHOLD) + types = request.data.get('type', settings.TYPES) + tags = request.data.get('tags') user = request.user - account = Member.objects.get(user=user).account - schedule_id = request.query_params.get('schedule_id') - site_id = request.query_params.get('site_id') + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # setting defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # scoped for sites + if site_ids: + for id in site_ids: + data = { + 'site_id': str(id), + 'configs': configs, + 'threshold': threshold, + 'type': types, + 'tags': tags, + 'user_id': str(user.id) + } + try: + # create test + res = create_test(**data) + if res['success']: + num_succeeded += 1 + succeeded.append(str(id)) + else: + num_failed += 1 + this_status = False + failed.append(str(id)) + print(res['reason']) + except Exception as e: + print(e) + if str(id) not in failed: + num_failed += 1 + this_status = False + failed.append(str(id)) + + # scoped for pages + if page_ids: + for id in page_ids: + data = { + 'page_id': str(id), + 'configs': configs, + 'threshold': threshold, + 'type': types, + 'tags': tags, + 'user_id': str(user.id) + } + try: + # create test + res = create_test(**data) + if res['success']: + num_succeeded += 1 + succeeded.append(str(id)) + else: + num_failed += 1 + this_status = False + failed.append(str(id)) + print(res['reason']) + except Exception as e: + print(e) + if str(id) not in failed: + num_failed += 1 + this_status = False + failed.append(str(id)) + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '201') + return Response(data, status=status.HTTP_201_CREATED) + - if schedule_id != None: - try: - schedule = Schedule.objects.get(id=schedule_id) - except: - data = {'reason': 'cannot find a Schedule with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - if schedule.site.account != user or schedule.account != account: - data = {'reason': 'retrieve Schedules of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) +def get_tests(request: object=None) -> object: + """ + Get one or more `Tests`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + test_id = request.query_params.get('test_id') + page_id = request.query_params.get('page_id') + lean = request.query_params.get('lean') + user = request.user + member = Member.objects.get(user=user) + + # deciding on scope + id = test_id if test_id else page_id + id_type = 'page' if page_id else 'test' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='test', + action='add',id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single test + if test_id != None: - serializer_context = {'request': request,} - serialized = ScheduleSerializer(schedule, context=serializer_context) + # get test + test = Test.objects.get(id=test_id) + + # serialize and return + serialized = TestSerializer(test, context={'request': request}) data = serialized.data record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) + # get all page scoped tests + page = Page.objects.get(id=page_id) + tests = Test.objects.filter(page=page).order_by('-time_created') - try: - site = Site.objects.get(id=site_id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - if site.account != account: - data = {'reason': 'retrieve Schedules of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - schedules = Schedule.objects.filter(site=site).order_by('-time_created') - + # serialize and return paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(schedules, request) - serializer_context = {'request': request,} - serialized = ScheduleSerializer(result_page, many=True, context=serializer_context) + result_page = paginator.paginate_queryset(tests, request) + serialized = TestSerializer(result_page, many=True, context={'request': request}) + if str(lean).lower() == 'true': + serialized = SmallTestSerializer(result_page, many=True, context={'request': request}) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response @@ -1147,333 +2653,4323 @@ def get_schedules(request): -def delete_schedule(request, id): +def get_test(request: object=None, id: str=None) -> object: + """ + Get single `Test` from the passed "id" - try: - schedule = Schedule.objects.get(id=id) - except: - data = {'reason': 'cannot find a Schedule with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ - task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - site = schedule.site + # get user and account user = request.user - account = Member.objects.get(user=user).account - - if site.account != account: - data = {'reason': 'delete Schedules you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - schedule.delete() - task.delete() + member = Member.objects.get(user=user) + account = member.account - data = {'message': 'Schedule has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='test', + action='get', id=id, id_type='test' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + # get test if checks passed + test = Test.objects.get(id=id) + + # serialize and return + serialized = TestSerializer(test, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) +def get_test_lean(request: object=None, id: str=None) -> object: + """ + Get a single `Test` and only return scores & timestamps + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ -def create_or_update_automation(request): + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='test', + action='get', id=id, id_type='test' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get test if checks passed + test = Test.objects.get(id=id) - try: - schedule = Schedule.objects.get(id=request.data.get('schedule_id')) - try: - automation = Automation.objects.get(id=schedule.automation.id) - if automation.account != account and automation.account != None: - data = {'reason': 'update a Automation you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - except: - automation = None - if schedule.account != account and schedule.account != None: - data = {'reason': 'create a Automation of a Schedule you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - except: - schedule = None - automation = None - - # get data - name = request.data.get('name') - expressions = request.data.get('expressions') - actions = request.data.get('actions') + # get images_delta if exists + images_delta = {"average_score": test.images_delta.get('average_score')} - if automation: - automation.name = name - automation.expressions = expressions - automation.actions = actions - automation.schedule = schedule - automation.save() - - if not automation: - automation = Automation.objects.create( - name=name, expressions=expressions, actions=actions, - schedule=schedule, user=request.user, account=account - ) + # get lighthouse_delta if exists + lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')} - if schedule: - schedule.automation = automation - schedule.save() - # update associated periodicTask - task = PeriodicTask.objects.get(id=schedule.periodic_task_id) - arguments = { - 'site_id': str(schedule.site.id), - 'automation_id': str(automation.id), - 'configs': json.loads(task.kwargs).get('configs', None), - 'type': json.loads(task.kwargs).get('type', None), - 'case_id': json.loads(task.kwargs).get('case_id', None), - 'updates': json.loads(task.kwargs).get('updates', None) - } - task.kwargs=json.dumps(arguments) - task.save() + # get lighthouse_delta if exists + yellowlab_delta = {"scores": test.yellowlab_delta['scores']} + + # format data + data = { + "id": str(test.id), + "site": str(test.site.id), + "tags": test.tags, + "type": test.type, + "time_created": str(test.time_created), + "time_completed": str(test.time_completed), + "pre_scan": str(test.pre_scan.id), + "post_scan": str(test.post_scan.id), + "score": test.score, + "lighthouse_delta": lighthouse_delta, + "yellowlab_delta": yellowlab_delta, + "images_delta": images_delta, + } - serializer_context = {'request': request,} - data = AutomationSerializer(automation, context=serializer_context).data + # return record_api_call(request, data, '200') response = Response(data, status=status.HTTP_200_OK) return response - -def get_automations(request): - automation_id = request.query_params.get('automation_id') - user = request.user - account = Member.objects.get(user=user).account - if automation_id != None: - try: - automation = Automation.objects.get(id=automation_id) - except: - data = {'reason': 'cannot find a Automation with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - if automation.account != account: - data = {'reason': 'retrieve an Automation you do not own',} - return Response(data, status=status.HTTP_403_FORBIDDEN) - serializer_context = {'request': request,} - serialized = AutomationSerializer(automation, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) +def delete_test(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Test` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object, - automations = Automation.objects.filter(user=user).order_by('-time_created') - paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(automations, request) - serializer_context = {'request': request,} - serialized = AutomationSerializer(result_page, many=True, context=serializer_context) - response = paginator.get_paginated_response(serialized.data) - record_api_call(request, response.data, '200') - return response + Returns: + HTTP Response object + """ + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='test', + action='delete', id=id, id_type='test' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get test if checks passed + test = Test.objects.get(id=id) + + # remove s3 objects + delete_test_s3_bg.apply_async(kwargs={'test_id': str(test.id), 'site_id': str(test.site.id), 'page_id': str(test.page.id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # delete test + page_id = str(test.page.id) + test.delete() + + # update site and page with most recent data + update_site_and_page_info.apply_async( + kwargs={'resource': 'test', 'page_id': str(page_id)}, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # return response + data = {'message': 'Test deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def delete_many_tests(request: object=None) -> object: + """ + Deletes one or more `Tests` associated + with the passed "request.ids" + + Args: + 'request' : object, + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check for ids + if ids is not None: + + # setting defaults + count = len(ids) + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + user = request.user + this_status = True + + # loop through passed ids + for id in ids: + + # trying to delete test + try: + # delete test and all assocaited resourses + data = delete_test(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + except Exception as e: + # add to failed attempts + print(e) + num_failed += 1 + failed.append(str(id)) + this_status = False + + # format data + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + + # return response + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + # return error + data = { + 'reason': 'you must provide an array of id\'s' + } + record_api_call(request, data, '400') + response = Response(data, status=status.HTTP_400_BAD_REQUEST) + return response + + + + +def get_tests_zapier(request: object=None) -> object: + """ + Get all `Tests` associated with user's Account. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + member = Member.objects.get(user=request.user) + account = member.account + page_id = request.query_params.get('page_id') + site_id = request.query_params.get('site_id') + id = page_id if page_id else site_id + id_type = 'page' if page_id else 'site' + _status = request.query_params.get('status') + tests = None + + # deciding on scope + resource = 'test' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all page associated tests + if page_id: + tests = Test.objects.filter( + page__account=account, + page__id=page_id, + ).exclude( + time_completed=None, + pre_scan=None, + post_scan=None, + ).order_by('-time_created') + + # get all site associated tests + if site_id: + tests = Test.objects.filter( + site__account=account, + site__id=site_id, + ).exclude( + time_completed=None, + pre_scan=None, + post_scan=None, + ).order_by('-time_created') + + # get all account assocoiated tests + if tests is None: + tests = Test.objects.filter( + site__account=account, + ).exclude( + time_completed=None, + pre_scan=None, + post_scan=None, + ).order_by('-time_created') + + # filter my status if requested + if status is not None: + tests = tests.filter(status=_status) + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + tests = tests.filter(site__id__in=id_list).order_by('-time_created') + + # build response data + data = [] + + for test in tests: + data.append({ + 'id' : str(test.id), + 'page' : str(test.page.id), + 'site' : str(test.site.id), + 'pre_scan' : str(test.pre_scan.id) if test.pre_scan else None, + 'post_scan' : str(test.post_scan.id) if test.post_scan else None, + 'time_created' : str(test.time_created), + 'time_completed' : str(test.time_completed), + 'type' : test.type, + 'status' : str(test.status), + 'score' : test.score, + 'threshold' : test.threshold, + 'component_scores' : test.component_scores, + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin Issue Services ------ ### + + + +def create_or_update_issue(request: object=None, **kwargs) -> object: + """ + Creates or Updates an `Issue` + + Args: + 'request': object + 'kwargs': dict + + Returns: + HTTP Response object + """ + + # get request data + if request is not None: + id = request.data.get('id') + trigger = request.data.get('trigger') + title = request.data.get('title') + details = request.data.get('details') + _status = request.data.get('status') + affected = request.data.get('affected') + labels = request.data.get('labels') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # get kwargs data + if request is None: + id = kwargs.get('id') + trigger = kwargs.get('trigger') + title = kwargs.get('title') + details = kwargs.get('details') + _status = kwargs.get('status') + affected = kwargs.get('affected') + labels = kwargs.get('labels') + account_id = kwargs.get('account_id') + user_id = kwargs.get('user_id') + user = User.objects.get(id=user_id) + member = Member.objects.get(user=user) + account = Account.objects.get(id=account_id) + + # decide on action + action = 'update' if id else 'add' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='issue', + action=action, id=id, id_type='issue' + ) + if not check_data['allowed']: + data = { + 'reason': check_data['error'], + 'success': False, + 'code': check_data['code'], + 'status': check_data['status'] + } + if request is not None: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get Issue if id is present + if id is not None: + issue = Issue.objects.get(id=id) + + # update data + if trigger is not None: + issue.trigger = trigger + if title is not None: + issue.title = title + if details is not None: + issue.details = details + if _status is not None: + issue.status = _status + if affected is not None: + issue.affected = affected + if labels is not None: + issue.labels = labels + + # save new data + issue.save() + + # create new Issue + if id is None: + issue = Issue.objects.create( + account = account, + title = title, + details = details, + labels = labels, + trigger = trigger, + affected = affected + ) + + # decide on response type + if request is not None: + # serialize and return + serialized = IssueSerializer(issue, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # return object response + data = { + 'success': True, + 'issue': issue, + } + return data + + + + +def generate_issue(request: object=None, **kwargs) -> object: + """ + Generates a new `Issue` based on the data + passed in the request or kwargs + + Args: + 'request': object + 'kwargs': dict + + Returns: + HTTP Response object + """ + + # get request data + if request is not None: + id = request.data.get('id') + trigger = request.data.get('trigger') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # get kwargs data + if request is None: + id = kwargs.get('id') + trigger = kwargs.get('trigger') + account_id = kwargs.get('account_id') + user_id = kwargs.get('user_id') + user = User.objects.get(id=user_id) + member = Member.objects.get(user=user) + account = Account.objects.get(id=account_id) + + # decide on action + action = 'add' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='issue', + action=action, id_type='issue' + ) + if not check_data['allowed']: + data = { + 'reason': check_data['error'], + 'success': False, + 'code': check_data['code'], + 'status': check_data['status'] + } + if request is not None: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # decide on which Issue type + # based on trigger + if trigger == 'scan': + if Scan.objects.filter(id=id, site__account=account).exists(): + scan = Scan.objects.get(id=id) + I = Issuer(scan=scan) + else: + data = { + 'reason': 'Scan not found', + 'success': False, + 'code': 404, + 'status': status.HTTP_404_NOT_FOUND + } + if request is not None: + record_api_call(request, data, 404) + return Response(data, status=status.HTTP_404_NOT_FOUND) + return data + + if trigger == 'test': + if Test.objects.filter(id=id, site__account=account).exists(): + test = Test.objects.get(id=id) + I = Issuer(test=test) + else: + data = { + 'reason': 'Test not found', + 'success': False, + 'code': 404, + 'status': status.HTTP_404_NOT_FOUND + } + if request is not None: + record_api_call(request, data, 404) + return Response(data, status=status.HTTP_404_NOT_FOUND) + return data + + if trigger == 'caserun': + if CaseRun.objects.filter(id=id, site__account=account).exists(): + caserun = CaseRun.objects.get(id=id) + I = Issuer(caserun=caserun) + else: + data = { + 'reason': 'CaseRun not found', + 'success': False, + 'code': 404, + 'status': status.HTTP_404_NOT_FOUND + } + if request is not None: + record_api_call(request, data, 404) + return Response(data, status=status.HTTP_404_NOT_FOUND) + return data + + # create new Issue + issue = I.build_issue() + + # decide on response type + if request is not None: + # serialize and return + serialized = IssueSerializer(issue, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # return object response + data = { + 'success': True, + 'issue': issue, + } + return data + + + + +def update_many_issues(request: object=None) -> object: + """ + Updates many `Issues` passed in a list + + Args: + 'ids' : list + 'updates' : dict + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + updates = request.data.get('updates') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # set defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # loop through ids and update + for id in ids: + # reformat update data + data = updates + data['id'] = str(id) + data['account_id'] = str(account.id) + data['user_id'] = str(user.id) + + # send update + try: + data = create_or_update_issue(**data) + if data.get('reason'): + raise Exception + + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + + except Exception as e: + print(e) + if str(id) not in failed: + num_failed += 1 + this_status = False + failed.append(str(id)) + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def get_issues(request: object=None) -> object: + """ + Get one or more `Issues`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + issue_id = request.query_params.get('issue_id') + site_id = request.query_params.get('site_id') + page_id = request.query_params.get('page_id') + sort = request.query_params.getlist('sort') + + user = request.user + member = Member.objects.get(user=user) + account = member.account + issues = None + + # deciding on scope + resource = 'issue' + id = issue_id if issue_id else (site_id if site_id else page_id) + id_type = 'issue' if issue_id else ('site' if site_id else 'page') + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, action='get', + id=id, id_type=id_type, + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # issue-specific sorting dict + sorting_items = { + # accend + 'time_created': 'time_created', + 'status': 'status', + 'title': 'title', + 'affected': 'affected__str', + # decend + '-time_created': '-time_created', + '-status': '-status', + '-title': '-title', + '-affected': '-affected__str', + } + + # transform sort param + _sort = [] + for s in sort: + _sort.extend(s.split(',')) + + # build order_by list + ordering = [sorting_items[s] for s in _sort if s in sorting_items] + if not ordering: + ordering = ['-time_created'] + + # get single issue + if issue_id != None: + + # get test + issue = Issue.objects.get(id=issue_id) + + # serialize and return + serialized = IssueSerializer(issue, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get all issues scoped page if page_id passed + if page_id is not None: + issues = Issue.objects.filter( + affected__icontains={'id': page_id}, + account=account + ).order_by('-status', '-time_created') + # get all issues scoped page if page_id passed + if site_id is not None: + issues = Issue.objects.filter( + affected__icontains={'id': site_id}, + account=account + ).order_by(*ordering) + + # get all account assocoiated issues + if issues is None: + issues = Issue.objects.filter( + account=account + ).order_by(*ordering) + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + new_ids = id_list + for id in id_list: + for page in Page.objects.filter(site__id=id): + new_ids.append(str(page.id)) + issues = issues.filter(affected__id__in=new_ids).order_by(*ordering) + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(issues, request) + serialized = IssueSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_issue(request: object=None, id: str=None) -> object: + """ + Get single `Issue` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='issue', action='get', + id=id, id_type='issue' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get issue if checks passed + issue = Issue.objects.get(id=id) + + # serialize and return + serialized = IssueSerializer(issue, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def search_issues(request: object=None) -> object: + """ + Searches for matching `Issues` to the passed + "query" + + Args: + 'request': obejct + + Returns: + HTTP Response object + """ + + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + query = request.query_params.get('query') + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='issue', action='get' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + + # search for issues + issues = Issue.objects.filter( + Q(account=account, title__icontains=query) | + Q(account=account, details__icontains=query) | + Q(account=account, affected__icontains=query) + ).order_by('-status', '-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + new_ids = id_list + for id in id_list: + for page in Page.objects.filter(site__id=id): + new_ids.append(str(page.id)) + issues = issues.filter(affected__id__in=new_ids).order_by('-time_created') + + # serialize and rerturn + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(issues, request) + serialized = IssueSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def delete_issue(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Issue` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='issue', action='delete', + id=id, id_type='issue', + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get issue if checks passed + issue = Issue.objects.get(id=id) + + # delete test + issue.delete() + + # return response + data = {'message': 'Issue deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def delete_many_issues(request: object=None) -> object: + """ + Deletes many `Issues` passed in a list + + Args: + 'ids': list + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # set defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # loop through ids and delete + for id in ids: + + # trying to delete issue + try: + # delete issue and all assocaited resourses + data = delete_issue(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + except Exception as e: + # add to failed attempts + print(e) + num_failed += 1 + failed.append(str(id)) + this_status = False + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def get_issues_zapier(request: object=None) -> object: + """ + Get all `Issues` associated with user's Account. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + member = Member.objects.get(user=request.user) + account = member.account + page_id = request.query_params.get('page_id') + site_id = request.query_params.get('site_id') + id = page_id if page_id else site_id + id_type = 'page' if page_id else 'site' + issues = None + + # deciding on scope + resource = 'issue' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all page associated issues + if page_id: + issues = Issue.objects.filter( + account=account, + affected__icontains=page_id, + ).order_by('-status','-time_created') + + # get all site associated issues + if site_id: + issues = Issue.objects.filter( + account=account, + affected__icontains=site_id, + ).order_by('-status', '-time_created') + + # get all account assocoiated issues + if issues is None: + issues = Issue.objects.filter( + account=account + ).order_by('-status', '-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + new_ids = id_list + for id in id_list: + for page in Page.objects.filter(site__id=id): + new_ids.append(str(page.id)) + issues = issues.filter(affected__id__in=new_ids).order_by('-time_created') + + # build response data + data = [] + + for issue in issues: + data.append({ + 'id' : str(issue.id), + 'title' : str(issue.title), + 'time_created' : str(issue.time_created), + 'details' : str(issue.details), + 'trigger' : issue.trigger, + 'affected' : issue.affected, + 'labels' : issue.labels, + 'status' : str(issue.status), + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin Schedule Services ------ ### + + + + +def create_or_update_schedule(request: object=None, **kwargs) -> object: + """ + Creates or Updates a `Schedule` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + if request: + schedule_status = request.data.get('status') + begin_date_raw = request.data.get('begin_date') + time = request.data.get('time') + timezone = request.data.get('timezone') + freq = request.data.get('frequency') + task_type = request.data.get('task_type') + types = request.data.get('type', settings.TYPES) + configs = request.data.get('configs', None) + threshold = request.data.get('threshold', settings.TEST_THRESHOLD) + schedule_id = request.data.get('schedule_id') + resources = request.data.get('resources') + tags = request.data.get('tags') + scope = request.data.get('scope') + case_id = request.data.get('case_id') + flow_id = request.data.get('flow_id') + updates = request.data.get('updates') + user = request.user + + if not request: + schedule_status = kwargs.get('status') + begin_date_raw = kwargs.get('begin_date') + time = kwargs.get('time') + timezone = kwargs.get('timezone') + freq = kwargs.get('frequency') + task_type = kwargs.get('task_type') + types = kwargs.get('type', settings.TYPES) + configs = kwargs.get('configs', None) + threshold = kwargs.get('threshold', settings.TEST_THRESHOLD) + schedule_id = kwargs.get('schedule_id') + resources = kwargs.get('resources') + tags = kwargs.get('tags') + scope = kwargs.get('scope') + case_id = kwargs.get('case_id') + flow_id = kwargs.get('flow_id') + updates = kwargs.get('updates') + user_id = kwargs.get('user_id') + user = User.objects.get(id=user_id) + + # get account + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # setting defaults + schedule = None + + # deciding on action type + action = 'add' if not schedule_id else 'update' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='schedule', + action=action, id=schedule_id, id_type='schedule' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get schedule if checks passed and id is present + if schedule_id: + schedule = Schedule.objects.get(id=schedule_id) + + # toggling schedule status + if schedule_status != None and schedule != None: + # update task + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + if schedule_status == 'Paused': + task.enabled = False + if schedule_status == 'Active': + task.enabled = True + # update schedule + schedule.status = schedule_status + task.save() + schedule.save() + + # creating or updating schedule + if not schedule_status: + + # get alert if schedule exists + alert_id = None + if schedule: + if Alert.objects.filter(schedule=schedule).exists(): + alert = Alert.objects.filter(schedule=schedule)[0] + alert_id = str(alert.id) + + # build task + task = f'api.tasks.create_{task_type}_bg' + + # build args + arguments = { + 'scope': scope, + 'resources': resources, + 'tags': tags, + 'account_id': str(account.id), + 'updates': updates, + 'configs': configs, + 'case_id': case_id, + 'flow_id': flow_id, + 'type': types, + 'threshold': threshold, + 'alert_id': alert_id, + '_queue': getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'), + } + + # setting start date default + begin_date = datetime.now() + + # parsing begin date + if begin_date_raw: + # begin_date = datetime.strptime(begin_date_raw, '%Y-%m-%d %H:%M:%S.%f') + begin_date = datetime.fromisoformat(begin_date_raw[:-1] + '+00:00') + + # building cron expression time & date + num_day_of_week = begin_date.weekday() + day = begin_date.strftime("%d") + minute = time[3:5] + hour = time[0:2] + + # building cron expression freq + if freq == '10-min': + minute = '*/10' + hour = '*' + day_of_week = '*' + day_of_month = '*' + elif freq == '30-min': + minute = '*/30' + hour = '*' + day_of_week = '*' + day_of_month = '*' + elif freq == 'hourly': + hour = '*/1' + day_of_week = '*' + day_of_month = '*' + elif freq == 'daily': + day_of_week = '*' + day_of_month = '*' + elif freq == 'weekly': + day_of_week = num_day_of_week + day_of_month = '*' + elif freq == 'monthly': + day_of_week = '*' + day_of_month = day + + # create unique str for + rand_str = secrets.token_urlsafe(6) + + # building unique task name + task_name = f'{task_type}_{scope}_{rand_str}_{freq}_@{time}_{account.user.id}' + + # building or updating crontab + crontab, _ = CrontabSchedule.objects.get_or_create( + timezone=timezone, + minute=minute, + hour=hour, + day_of_week=day_of_week, + day_of_month=day_of_month, + ) + + # updating periodic task if schedule + periodic_task = None + if schedule: + if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists(): + # update existing task + periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id) + + # grabbing task_id + arguments['task_id'] = str(periodic_task[0].id) + + # updating task with args + periodic_task.update( + crontab=crontab, + name=task_name, + task=task, + kwargs=json.dumps(arguments), + queue=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'), + routing_key=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'), + ) + # get periodic task by id + periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + + # check if no task yet + if not periodic_task: + + # check if task exists + if PeriodicTask.objects.filter(name=task_name).exists(): + data = {'reason': 'Schedule already exists', 'code': '401'} + if request: + record_api_call(request, data, '401') + return Response(data, status=status.HTTP_401_UNAUTHORIZED) + return data + + # create new periodic task + periodic_task = PeriodicTask.objects.create( + crontab=crontab, + name=task_name, + task=task, + queue=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'), + routing_key=getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled'), + ) + + # inserting task_id + arguments['task_id'] = str(periodic_task.id) + + # updating args + periodic_task.kwargs = json.dumps(arguments) + periodic_task.save() + + # building extras for scheduls + extras = { + "configs": configs, + "type": types, + "case_id": case_id, + "flow_id": flow_id, + "updates": updates, + "threshold": threshold, + } + + # update existing schedule + if schedule: + + # update each param if passed + if timezone: + schedule.timezone = timezone + if begin_date: + schedule.begin_date = begin_date + if time: + schedule.time = time + if freq: + schedule.frequency = freq + if task: + schedule.task = task + if crontab: + schedule.crontab_id = crontab.id + if task_type: + schedule.task_type = task_type + if extras: + schedule.extras = extras + if resources is not None: + schedule.resources = resources + if tags is not None: + schedule.tags = tags + + # save udpdates + schedule.save() + + # create new schedule + if not schedule: + schedule = Schedule.objects.create( + user=request.user, + scope=scope, + resources=resources, + tags=tags, + task_type=task_type, + timezone=timezone, + begin_date=begin_date, + time=time, + frequency=freq, + task=task, + crontab_id=crontab.id, + periodic_task_id=periodic_task.id, + extras=extras, + account=account + ) + + # updated accounts usage + account.usage['schedules'] += 1 + account.save() + + # deciding on response type + if request: + # serialize and return + data = ScheduleSerializer(schedule, context={'request': request}).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + # return object response + data = { + 'success': True, + 'schedule': schedule, + } + return data + + + + +def update_many_schedules(request: object=None) -> object: + """ + Updates many `Schedules` passed in a list + + Args: + 'ids' : list + 'updates' : dict + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + updates = request.data.get('updates') + member = Member.objects.get(user=request.user) + account = member.account + + # set defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # loop through ids and update + for id in ids: + # reformat update data + data = updates + data['schedule_id'] = str(id) + data['user_id'] = str(request.user.id) + + # send update + try: + data = create_or_update_schedule(**data) + if data.get('reason'): + raise Exception(data['reason']) + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + + except Exception as e: + print(e) + if str(id) not in failed: + num_failed += 1 + this_status = False + failed.append(str(id)) + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def run_schedule(request: object=None) -> object: + """ + Grabs all the args from the asociated perodic_task + and executes the task manually without interupting + the perodic_task's normal cycle. + + Args: + requests: object + + + Returns: + Response + """ + + # get request data + schedule_id = request.data.get('schedule_id') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='schedule', + action='get', id=schedule_id, id_type='schedule' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get schedule and assocated task if checks passed + schedule = Schedule.objects.get(id=schedule_id) + task = schedule.task_type + perodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + task_kwargs = json.loads(perodic_task.kwargs) + queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + task_kwargs['_queue'] = queue + + # check location + local = schedule.extras['configs'].get('location', settings.LOCATION) + location_data = check_location(request, local) + if location_data['routed']: + return location_data['response'] + + # decidign on which task + if task == 'scan': + # run create_scan_bg + create_scan_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue) + if task == 'test': + # run create_test_bg + create_test_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue) + if task == 'caserun': + # run create_caserun_bg + create_caserun_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue) + if task == 'flowrun': + # run create_flowrun_bg + create_flowrun_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue) + if task == 'report': + # run create_report_bg + create_report_bg.apply_async(kwargs=task_kwargs, queue=queue, routing_key=queue) + + # serialize and return + data = ScheduleSerializer(schedule, context={'request': request}).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_schedules(request: object=None) -> object: + """ + Get one or more `Schedules`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + schedule_id = request.query_params.get('schedule_id') + scope = request.query_params.get('scope') + resource_id = request.query_params.get('resource_id') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # setting default + schedules = None + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='schedule', + action='get', id=schedule_id, id_type='schedule' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single schedule + if schedule_id: + + # get schedule + schedule = Schedule.objects.get(id=schedule_id) + + # serialize and return + serialized = ScheduleSerializer(schedule, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get all account scoped schedules + if scope == 'account': + schedules = Schedule.objects.filter( + account=account, + scope='account' + ).order_by('-time_created') + + # get all non account scoped + if scope != 'account' and resource_id is None: + schedules = Schedule.objects.filter( + account=account, + scope=scope + ).order_by('-time_created') + + # get all non account scoped schedules with resource_id + if scope != 'account' and resource_id: + schedules = Schedule.objects.filter( + account=account, + resources__icontains=resource_id, + scope=scope + ).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(schedules, request) + serialized = ScheduleSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_schedule(request: object=None, id: str=None) -> object: + """ + Get single `Schedule` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='schedule', + action='get', id=id, id_type='schedule' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get schedule if checks passed + schedule = Schedule.objects.get(id=id) + + # serialize and return + serialized = ScheduleSerializer(schedule, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_schedule(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Schedule` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object + + Returns: + HTTP Response object + """ + + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='schedule', + action='delete', id=id, id_type='schedule' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get schedule and task if checks passed + schedule = Schedule.objects.get(id=id) + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + + # delete schedule + schedule.delete() + + # delete task + task.delete() + + # decrement resource + decrement_resource(account=account, resource='schedules') + + # return response + data = {'message': 'Schedule deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def delete_many_schedules(request: object=None) -> object: + """ + Deletes many `Schedules` passed in a list + + Args: + 'ids': list + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # set defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # loop through ids and delete + for id in ids: + + # trying to delete schedule + try: + # delete issue and all assocaited resourses + data = delete_schedule(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + except Exception as e: + # add to failed attempts + print(e) + num_failed += 1 + failed.append(str(id)) + this_status = False + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def delete_tasks_and_schedules( + resource_id : str=None, + scope : object=None, + account : object=None + ) -> None: + """ + Helper function to delete any `Schedules` & `PerodicTasks` + associated with the passed "resource_id", "scope", and + "account" + + Args: + 'resource_id' : str, + 'scope' : str + 'account' : object + + Returns: None + """ + # get all scopped Schedules + schedules = Schedule.objects.filter( + resources__icontains=resource_id, + account=account, + scope=scope + ) + + # remove any associated tasks + for schedule in schedules: + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + try: + task.delete() + except Exception as e: + print(e) + + # delete Schedules + schedules.delete() + + return None + + + + +### ------ Begin Alert Services ------ ### + + + + +def create_or_update_alert(request: object=None) -> object: + """ + Creates or Updates an `Alert` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + actions = request.data.get('actions') + schedule_id = request.data.get('schedule_id') + alert_id = request.data.get('alert_id') + name = request.data.get('name') + expressions = request.data.get('expressions') + + # set defaults + alert = None + schedule = None + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # deciding on recsource + id = alert_id if alert_id else schedule_id + id_type = 'alert' if alert_id else 'schedule' + action = 'add' if schedule_id else 'update' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='alert', + action=action, id=id, id_type=id_type, + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get schedule if checks passed + if schedule_id: + schedule = Schedule.objects.get(id=schedule_id) + if alert_id: + alert = Alert.objects.get(id=alert_id) + schedule = alert.schedule + + # update existing alert + if alert: + if name: + alert.name = name + if expressions: + alert.expressions = expressions + if actions: + alert.actions = actions + if schedule: + alert.schedule = schedule + # save updates + alert.save() + + # create new alert + if not alert: + alert = Alert.objects.create( + name=name, + expressions=expressions, + actions=actions, + schedule=schedule, + user=user, + account=account + ) + + # update schedule + if schedule: + + # update schedule with new alert + schedule.alert = alert + schedule.save() + + # update associated periodicTask + task = PeriodicTask.objects.get(id=schedule.periodic_task_id) + + # update periodic task + arguments = { + 'scope': json.loads(task.kwargs).get('scope'), + 'resources': json.loads(task.kwargs).get('resources'), + 'account_id': json.loads(task.kwargs).get('account_id'), + 'alert_id': str(alert.id), + 'configs': json.loads(task.kwargs).get('configs'), + 'type': json.loads(task.kwargs).get('type'), + 'threshold': json.loads(task.kwargs).get('threshold'), + 'case_id': json.loads(task.kwargs).get('case_id'), + 'flow_id': json.loads(task.kwargs).get('flow_id'), + 'updates': json.loads(task.kwargs).get('updates'), + 'task_id': json.loads(task.kwargs).get('task_id'), + } + task.kwargs=json.dumps(arguments) + task.save() + + # serialize and return + data = AlertSerializer(alert, context={'request': request}).data + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_alerts(request: object=None) -> object: + """ + Get one or more `Alerts`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + alert_id = request.query_params.get('alert_id') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='alert', + action='get', id=alert_id, id_type='alert' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single alert + if alert_id: + + # get alert + alert = Alert.objects.get(id=alert_id) + + # serialize and return + serialized = AlertSerializer(alert, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get all alerts associated with account + alerts = Alert.objects.filter(account=account).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(alerts, request) + serialized = AlertSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_alert(request: object=None, id: str=None) -> object: + """ + Get single `Alert` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='alert', + action='get', id=id, id_type='alert' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get alert if checks passed + alert = Alert.objects.get(id=id) + + # serialize and return + serialized = AlertSerializer(alert, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_alert(request: object=None, id: str=None) -> object: + """ + Deletes the `Alert` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account info + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='alert', + action='delete', id=id, id_type='alert' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get alert if checks passed + alert = Alert.objects.get(id=id) + + # delete alert + alert.delete() + + # return response + data = {'message': 'Alert deleted'} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin Report Services ------ ### + + + + +def create_or_update_report(request: object=None) -> object: + """ + Creates or Updates an `Report` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + report_id = request.data.get('report_id') + site_id = request.data.get('site_id') + report_type = request.data.get('type') + lookback_days = request.data.get('lookback_days') + text_color = request.data.get('text_color', '#24262d') + background_color = request.data.get('background_color', '#e1effd') + highlight_color = request.data.get('highlight_color', '#4283f8') + + # set defaults + report = None + site = None + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + id = report_id if report_id else site_id + id_type = 'report' if report_id else 'site' + action = 'update' if report_id else 'add' + + # validate create/update payload shape + valid_types = ['issues', 'tests', 'caseruns', 'performance'] + if not report_id and not site_id: + data = {'reason': 'site_id is required when report_id is not provided'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + if lookback_days is None: + data = {'reason': 'lookback_days is required and must be one of 1, 7, 30, 90'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + try: + lookback_days = int(lookback_days) + except Exception: + data = {'reason': 'lookback_days must be one of 1, 7, 30, 90'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + if lookback_days not in [1, 7, 30, 90]: + data = {'reason': 'lookback_days must be one of 1, 7, 30, 90'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + if not isinstance(report_type, list) or len(report_type) == 0: + data = {'reason': 'type is required and must be a non-empty array'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + report_type = list(dict.fromkeys([str(item).strip().lower() for item in report_type if str(item).strip()])) + if not all(item in valid_types for item in report_type): + data = {'reason': f'type must be a subset of {valid_types}'} + record_api_call(request, data, '400') + return Response(data, status=status.HTTP_400_BAD_REQUEST) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='report', + action=action, id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get report if checks passed + if report_id: + report = Report.objects.get(id=report_id) + if site_id: + if not Site.objects.filter(id=site_id, account=account).exists(): + data = {'reason': 'site not found'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = Site.objects.get(id=site_id) + else: + site = report.site + # get site if creating + if not report: + if not Site.objects.filter(id=site_id, account=account).exists(): + data = {'reason': 'site not found'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = Site.objects.get(id=site_id) + + # build report info + info = { + "text_color": text_color, + "background_color": background_color, + "highlight_color": highlight_color, + "lookback_days": lookback_days, + "types": report_type, + } + + # update report + if report: + old_info = report.info if isinstance(report.info, dict) else {} + old_info.update(info) + report.info = old_info + report.type = report_type + if site: + report.site = site + report.page = None + # save updates + report.save() + + # create new report + if not report: + report = Report.objects.create( + user=request.user, + page=None, + site=site, + account=account, + info=info, + type=report_type + ) + + # get uncached report + un_cached_report = Report.objects.get(id=report.id) + + # generate report + report_data = R(report=un_cached_report).generate_report() + + # serialize report + new_report = ReportSerializer( + report_data['report'], + context={'request': request} + ).data + + # format return data + data = { + 'report': new_report, + 'success': report_data['success'], + 'message': report_data['message'] + } + + # serialize and return + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_reports(request: object=None) -> object: + """ + Get one or more `Reports`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + site_id = request.query_params.get('site_id') + report_id = request.query_params.get('report_id') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + id = report_id if report_id else site_id + id_type = 'report' if report_id else 'site' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='report', + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single report + if report_id: + + # get report + report = Report.objects.get(id=report_id) + + # serialize and return + serialized = ReportSerializer(report, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get reports scoped to site if checks passed + if site_id: + if not Site.objects.filter(id=site_id, account=account).exists(): + data = {'reason': 'site not found'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + site = Site.objects.get(id=site_id) + reports = Report.objects.filter(site=site, account=account).order_by('-time_created') + + # get reports scoped to user if checks passed + if site_id is None and report_id is None: + reports = Report.objects.filter(user=request.user).order_by('-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + reports = reports.filter(site__id__in=id_list).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(reports, request) + serialized = ReportSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_report(request: object=None, id: str=None) -> object: + """ + Get single `Report` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='report', + action='get', id=id, id_type='report' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get report if checks passed + report = Report.objects.get(id=id) + + # serialize and return + serialized = ReportSerializer(report, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_report(request: object=None, id: str=None) -> object: + """ + Deletes the `Report` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account info + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='report', + action='delete', id=id, id_type='report' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get report if checks passed + report = Report.objects.get(id=id) + + # remove s3 objects + delete_report_s3_bg.apply_async(kwargs={'report_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # remove report + report.delete() + + # return reponse + data = {'message': 'Report deleted'} + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def export_report(request: object=None) -> object: + """ + Used to create and send a Cursion.landing + `Report` to the passed "email" + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # getting data from request + report_id = request.data.get('report_id') + email = request.data.get('email') + first_name = request.data.get('first_name') + + # send task to background + create_report_export_bg.apply_async( + kwargs={'report_id': str(report_id), 'email': email, 'first_name': first_name}, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # building response + data = { + 'success': True, + 'error': None + } + + # returning response + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin Cases Services ------ ### + + + + +def create_or_update_case(request: object=None) -> object: + """ + Creates or Updates a `Case` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + case_id = request.data.get('case_id') + steps = request.data.get('steps') + site_url = request.data.get('site_url') + site_id = request.data.get('site_id') + title = request.data.get('title') + tags = request.data.get('tags') + _type = request.data.get('type') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # setting defaults + site = None + case = None + action = 'update' if case_id else 'add' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action=action, id=case_id, id_type='case' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get site if site_url passed + if site_url: + if Site.objects.filter(account=account, site_url=site_url).exists(): + site = Site.objects.filter(account=account, site_url=site_url)[0] + + # get site if site_id passed + if site_id: + if Site.objects.filter(account=account, id=site_id).exists(): + site = Site.objects.get(id=site_id) + site_url = site.site_url + + # check for no site and no case_id + if not site and not case_id: + data = {'reason': 'site not found'} + record_api_call(request, data, '404') + response = Response(data, status=status.HTTP_404_NOT_FOUND) + + # get case if checks passed + if case_id: + case = Case.objects.get(id=case_id) + + # update Case + if case: + if steps is not None: + steps_data = save_case_steps(steps, case_id) + case.steps = steps_data + if title is not None: + case.title = title + if tags is not None: + case.tags = tags + if site is not None: + case.site = site + if site_url is not None: + case.site_url = site_url + # save updates + case.save() + + # create Case + if not case: + + # generate new uuid + case_id = uuid.uuid4() + + # save step data in s3 + steps_data = save_case_steps(steps, case_id) + + # create new Case + case = Case.objects.create( + id = case_id, + user = user, + account = account, + title = title, + type = _type if _type is not None else "recorded", + site = site, + site_url = site_url, + steps = steps_data, + + ) + + # signals.py will pickup 'created' instance + # and run Caser().pre_run() in background + + # serialize and return + data = CaseSerializer(case, context={'request': request}).data + data['client'] = settings.CLIENT_URL_ROOT + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def save_case_steps(steps: dict, case_id: str) -> dict: + """ + Helper function that uploads the "steps" data to + s3 bucket + + Args: + 'steps' : dict, + 'case_id' : str + + Returns: + 'num_steps' : int, + 'url' : str + """ + + # setup boto3 configurations + s3 = boto3.client( + 's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), + aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), + region_name=str(settings.AWS_S3_REGION_NAME), + endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) + ) + + # saving as json file temporarily + steps_id = uuid.uuid4() + with open(f'{steps_id}.json', 'w') as fp: + json.dump(steps, fp) + + # seting up paths + steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json') + remote_path = f'static/cases/{case_id}/{steps_id}.json' + root_path = settings.AWS_S3_URL_PATH + steps_url = f'{root_path}/{remote_path}' + + # upload to s3 + with open(steps_file, 'rb') as data: + s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), + remote_path, ExtraArgs={ + 'ACL': 'public-read', + 'ContentType': 'application/json', + 'CacheControl': 'max-age=0' + } + ) + + # remove local copy + os.remove(steps_file) + + # format data + data = { + 'num_steps': len(steps), + 'url': steps_url + } + + # return response + return data + + + + +def case_pre_run(request: object=None, **kwargs) -> object: + """ + Inits case_pre_run_bg for the passed 'case_id' + + Expects: + case_id: str, + user_id: str, + + Returns: HTTP or Case object + """ + + # decide on data source + source = request.data if request else kwargs + + # get data + case_id = source.get('case_id') + user_id = source.get('user_id') if not request else request.user.id + + # get member, and account + member = Member.objects.get(user__id=user_id) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action='update', id=case_id, id_type='case' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get case + case = Case.objects.get(id=case_id) + + # create process obj + process = Process.objects.create( + site=case.site, + type='case.pre_run', + object_id=str(case.id), + account=case.account, + progress=1 + ) + + # start pre_run for new Case + case_pre_run_bg.apply_async( + kwargs={'case_id': str(case.id), 'process_id': str(process.id)}, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # return dynamic + if request: + serializer_context = {'request': request} + data = CaseSerializer(case, context={'request': request}).data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + return case + + + + +def get_cases(request: object=None) -> object: + """ + Get one or more `Cases`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + case_id = request.query_params.get('case_id') + site_id = request.query_params.get('site_id') + sort = request.query_params.getlist('sort') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # setting defaulta + case = None + site = None + id = case_id if case_id else site_id + id_type = 'case' if case_id else 'site' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # issue-specific sorting dict + sorting_items = { + # accend + 'time_created': 'time_created', + 'type': 'type', + 'title': 'title', + 'site': 'site_url', + # decend + '-time_created': '-time_created', + '-type': '-type', + '-title': '-title', + '-site': '-site_url', + } + + # transform sort param + _sort = [] + for s in sort: + _sort.extend(s.split(',')) + + # build order_by list + ordering = [sorting_items[s] for s in _sort if s in sorting_items] + if not ordering: + ordering = ['-time_created'] + + # get single case + if case_id: + + # get case + case = Case.objects.get(id=case_id) + + # serialize and return + serialized = CaseSerializer(case, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get site if checks passed + if site_id: + site = Site.objects.get(id=site_id) + + # get cases scoped by site + if site: + cases = Case.objects.filter(account=account, site=site).order_by(*ordering) + + # get cases scoped by account + if not site: + cases = Case.objects.filter(account=account).order_by(*ordering) + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + cases = cases.filter(site__id__in=id_list).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(cases, request) + serialized = CaseSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_case(request: object=None, id: str=None) -> object: + """ + Get single `Case` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action='get', id=id, id_type='case' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get case if checks passed + case = Case.objects.get(id=id) + + # serialize and return + serialized = CaseSerializer(case, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def search_cases(request: object=None) -> object: + """ + Searches for matching `Cases` to the passed + "query" + + Args: + 'request': obejct + + Returns: + HTTP Response object + """ + + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + query = request.query_params.get('query') + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', action='get' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # search for cases + cases = Case.objects.filter( + Q(account=account, title__icontains=query) | + Q(account=account, site_url__icontains=query) + ).order_by('-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + cases = cases.filter(site__id__in=id_list).order_by('-time_created') + + # serialize and rerturn + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(cases, request) + serialized = CaseSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def create_auto_cases(request: object=None) -> object: + """ + Initiates a new `Case` generation task for the `Site` + associated with either the passed "site_url" or "site_id" + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # check location + location_data = check_location(request, None) + if location_data['routed']: + return location_data['response'] + + # get request data + site_id = request.data.get('site_id') + site_url = request.data.get('site_url') + start_url = request.data.get('start_url') + max_cases = request.data.get('max_cases', 4) + max_layers = request.data.get('max_layers', 6) + configs = request.data.get('configs', None) + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # get site if only site_url present + if site_url is not None: + site = Site.objects.filter(account=account, site_url=site_url)[0] + site_id = str(site.id) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action='add', id=site_id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get site if only site_id present + if site_id and not site_url: + site = Site.objects.get(id=site_id) + + # create process obj + process = Process.objects.create( + site=site, + type='case.generate', + account=account, + progress=1 + ) + + # send data to bg_autocase_task + create_auto_cases_bg.apply_async( + kwargs={ + 'site_id': str(site_id), + 'process_id': str(process.id), + 'start_url': start_url, + 'configs': configs, + 'max_cases': max_cases, + 'max_layers': max_layers, + }, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # return response + data = { + 'message': 'Cases are generating', + 'process': str(process.id), + } + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def copy_case(request: object=None) -> object: + """ + Creates a copy of the passed `Case` + + Args: + 'request': object + + Returns: HTTP Response obejct + """ + + # get request data + case_id = request.data.get('case_id') + + # get user and acount + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action='add', id=case_id, id_type='case' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get case if checks passed + if case_id: + case = Case.objects.get(id=case_id, account=account) + + # download steps + steps = requests.get(case.steps['url']).json() + + # save steps as new s3 obj + new_case_id = uuid.uuid4() + steps_data = save_case_steps(steps, new_case_id) + + # create new case + new_case = Case.objects.create( + id = new_case_id, + user = user, + account = account, + title = f'Copy - {case.title}', + type = case.type, + site = case.site, + site_url = case.site_url, + steps = steps_data, + processed = True + + ) + + # return response + data = CaseSerializer(new_case, context={'request': request}).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def delete_case(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Case` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object, + + Returns: + HTTP Response object + """ + + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='case', + action='delete', id=id, id_type='case' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get case if checks passed + case = Case.objects.get(id=id) + + # delete case s3 objects + delete_case_s3_bg.apply_async(kwargs={'case_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # delete case + case.delete() + + # return response + data = {'message': 'Case deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def delete_many_cases(request: object=None) -> object: + """ + Deletes many `Cases` passed in a list + + Args: + 'ids': list + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # set defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # loop through ids and delete + for id in ids: + + # trying to delete case + try: + # delete case and all assocaited resourses + data = delete_case(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + except Exception as e: + # add to failed attempts + print(e) + num_failed += 1 + failed.append(str(id)) + this_status = False + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def get_cases_zapier(request: object=None) -> object: + """ + Get all `Cases` associated with user's Account. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + site_id = request.query_params.get('site_id') + cases = None + + # deciding on scope + resource = 'case' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, + action='get', id=site_id, id_type='site' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all site_id associated cases + if site_id: + site = Site.objects.get(id=site_id) + cases = Case.objects.filter( + account=account, + site=site + ).order_by('-time_created') + + # get all account assocoiated cases + if cases is None: + cases = Case.objects.filter( + account=account, + ).order_by('-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + cases = cases.filter(site__id__in=id_list).order_by('-time_created') + + # build response data + data = [] + + for case in cases: + data.append({ + 'id' : str(case.id), + 'title' : case.title, + 'time_created' : str(case.time_created), + 'site' : str(case.site.id), + 'site_url' : case.site_url, + 'steps' : case.steps, + 'tags' : case.tags + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin CaseRun Services ------ ### + + + + +def create_caserun(request: object=None) -> object: + """ + Creates a new `CaseRun` from the passed "case_id" for the + passed "site_id" + + Args: + 'request': obejct + + Returns: + HTTP Response object + """ + + # check location + location_data = check_location(request, None) + if location_data['routed']: + return location_data['response'] + + # get request data + case_id = request.data.get('case_id') + site_id = request.data.get('site_id') + updates = request.data.get('updates') + configs = request.data.get('configs', None) + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # updating configs if None: + configs = account.configs if configs == None else configs + + # check site + if not Site.objects.filter(id=site_id, account=account).exists(): + data = {'reason': 'site not found'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='caserun', + action='add', id=case_id, id_type='case' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get case & site if checks passed + case = Case.objects.get(id=case_id, account=account) + site = Site.objects.get(id=site_id, account=account) + + # getting steps from case + steps = requests.get(case.steps['url']).json() + + # adding new info to steps for caserun + for step in steps: + # expanding action + if step['action']['type'] != None: + step['action']['time_created'] = None + step['action']['time_completed'] = None + step['action']['exception'] = None + step['action']['status'] = None + step['action']['img'] = None + # expanding assertion + if step['assertion']['type'] != None: + step['assertion']['time_created'] = None + step['assertion']['time_completed'] = None + step['assertion']['exception'] = None + step['assertion']['status'] = None + + # updating values if requested + if updates != None: + for update in updates: + steps[int(update['index'])]['action']['value'] = update['value'] + + # update usage and meter resource + check_and_increment_resource(account.id, 'caseruns') + + # create new tescase + caserun = CaseRun.objects.create( + case = case, + title = case.title, + site = site, + user = request.user, + configs = configs, + steps = steps, + account = account + ) + + # pass the newly created CaseRun to the backgroud task to run + run_case.apply_async( + kwargs={ + 'caserun_id': str(caserun.id), + '_queue': ON_DEMAND_QUEUE + }, + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE + ) + + # serialize and return + data = { + 'id': str(caserun.id), + 'title': str(caserun.title), + 'site': str(site.id), + 'time_created': str(caserun.time_created) + } + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_caseruns(request: object=None) -> object: + """ + Get one or more `CaseRun`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + caserun_id = request.query_params.get('caserun_id') + site_id = request.query_params.get('site_id') + lean = request.query_params.get('lean') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # defaults + id = caserun_id if caserun_id else site_id + id_type = 'caserun' if caserun_id else 'site' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='caserun', + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single caserun + if caserun_id: + + # get caserun + caserun = CaseRun.objects.get(id=caserun_id) + + # serialize and return + serialized = CaseRunSerializer(caserun, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get caseruns scoped to site + if site_id: + site = Site.objects.get(id=site_id, account=account) + caseruns = CaseRun.objects.filter(site=site).order_by('-time_created') + + # get caseruns scoped to account + if not site_id: + caseruns = CaseRun.objects.filter(account=account).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(caseruns, request) + serialized = CaseRunSerializer(result_page, many=True, context={'request': request}) + if str(lean).lower() == 'true': + serialized = SmallCaseRunSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_caserun(request: object=None, id: str=None) -> object: + """ + Get single `CaseRun` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='caserun', + action='get', id=id, id_type='caserun' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get caserun if checks passed + caserun = CaseRun.objects.get(id=id) + + # serialize and return + serialized = CaseRunSerializer(caserun, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_caserun(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `CaseRun` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object + + Returns: + HTTP Response object + """ + + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='caserun', + action='delete', id=id, id_type='caserun' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get caserun if checks passed + caserun = CaseRun.objects.get(id=id) + + # remove s3 objects + delete_caserun_s3_bg.apply_async(kwargs={'caserun_id': str(id)}, queue=ON_DEMAND_QUEUE, routing_key=ON_DEMAND_QUEUE) + + # delete caserun + caserun.delete() + + # return response + data = {'message': 'CaseRun deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def get_caseruns_zapier(request: object=None) -> object: + """ + Get all `CaseRuns` associated with user's Account. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + _status = request.query_params.get('status') + user = request.user + member = Member.objects.get(user=user) + account = member.account + caseruns = None + + # deciding on scope + resource = 'caserun' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource=resource, + action='get', + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all account assocoiated caseruns + if caseruns is None: + caseruns = CaseRun.objects.filter( + account=account, + ).exclude( + time_completed=None, + ).order_by('-time_created') + + # filter by _status if requested + if _status is not None: + caseruns = caseruns.filter(status=_status) + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + caseruns = caseruns.filter(site__id__in=id_list).order_by('-time_created') + + # build response data + data = [] + + for caserun in caseruns: + data.append({ + 'id' : str(caserun.id), + 'case' : str(caserun.case.id), + 'title' : str(caserun.title), + 'site' : str(caserun.site.id), + 'time_created' : str(caserun.time_created), + 'time_completed' : str(caserun.time_completed), + 'configs' : caserun.configs, + 'status' : str(caserun.status), + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin Flow Services ------ ### + + + + +def create_or_update_flow(request: object=None) -> object: + """ + Creates or Updates a `Flow` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + flow_id = request.data.get('flow_id') + nodes = request.data.get('nodes') + edges = request.data.get('edges') + title = request.data.get('title') + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # setting defaults + flow = None + action = 'update' if flow_id else 'add' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action=action, id=flow_id, id_type='flow' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get flow if checks passed + if flow_id: + flow = Flow.objects.get(id=flow_id) + + # update flow + if flow: + if title is not None: + flow.title = title + if nodes is not None: + flow.nodes = nodes + if edges is not None: + flow.edges = edges + # save updates + flow.save() + + # create Case + if not flow: + + # create new Flow + flow = Flow.objects.create( + user = request.user, + account = account, + title = title if title is not None else 'Untitled Flow', + ) + + # serialize and return + data = FlowSerializer(flow, context={'request': request}).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_flows(request: object=None) -> object: + """ + Get one or more `Flows`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + flow_id = request.query_params.get('flow_id') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # setting default + flow = None + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action='get', id=flow_id, id_type='flow' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single flow + if flow_id: + + # get flow + flow = Flow.objects.get(id=flow_id) + + # serialize and return + serialized = FlowSerializer(flow, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get flows scoped by account + flows = Flow.objects.filter(account=account).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(flows, request) + serialized = FlowSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def get_flow(request: object=None, id: str=None) -> object: + """ + Get single `Flow` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action='get', id=id, id_type='flow' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get flow if checks passed + flow = Flow.objects.get(id=id) + + # serialize and return + serialized = FlowSerializer(flow, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def search_flows(request: object=None) -> object: + """ + Searches for matching `Flows` to the passed + "query" + + Args: + 'request': obejct + + Returns: + HTTP Response object + """ + + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + query = request.query_params.get('query') + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action='get' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # search for flows + flows = Flow.objects.filter( + Q(account=account, title__icontains=query) + ).order_by('-time_created') + + # serialize and rerturn + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(flows, request) + serialized = FlowSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response + + + + +def copy_flow(request: object=None) -> object: + """ + Creates a copy of the passed `Flow` + + Args: + 'request': object + + Returns: HTTP Response obejct + """ + + # get request data + flow_id = request.data.get('flow_id') + + # get user and acount + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action='add', id=flow_id, id_type='flow' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get flow if checks passed + if flow_id: + flow = Flow.objects.get(id=flow_id, account=account) + + # create new flow + new_flow = Flow.objects.create( + user = request.user, + account = account, + title = f'Copy - {flow.title}', + nodes = flow.nodes, + edges = flow.edges + ) + + # return response + data = FlowSerializer(new_flow, context={'request': request}).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def delete_flow(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Flow` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object, + + Returns: + HTTP Response object + """ + + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action='delete', id=id, id_type='flow' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get flow if checks passed + flow = Flow.objects.get(id=id) + + # delete flow + flow.delete() + + # return response + data = {'message': 'Flow deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + + + + +def delete_many_flows(request: object=None) -> object: + """ + Deletes many `Flows` passed in a list + + Args: + 'ids': list + + Returns: + HTTP Response object + """ + + # get request data + ids = request.data.get('ids') + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # set defaults + num_succeeded = 0 + succeeded = [] + num_failed = 0 + failed = [] + this_status = True + + # loop through ids and delete + for id in ids: + + # trying to delete flow + try: + # delete flow and all assocaited resourses + data = delete_flow(id=id, user=user) + if data.get('reason'): + raise Exception + + # add to success attempts + num_succeeded += 1 + succeeded.append(str(id)) + except Exception as e: + # add to failed attempts + print(e) + num_failed += 1 + failed.append(str(id)) + this_status = False + + # format and return + data = { + 'success': this_status, + 'num_succeeded': num_succeeded, + 'succeeded': succeeded, + 'num_failed': num_failed, + 'failed': failed, + } + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + + + +def get_flows_zapier(request: object=None) -> object: + """ + Get all `Flows` associated with user's Account. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + user = request.user + member = Member.objects.get(user=user) + account = member.account + flows = None + + # deciding on scope + resource = 'flow' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='flow', + action='get', + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all account assocoiated flows + if flows is None: + flows = Flow.objects.filter( + account=account, + ).order_by('-time_created') + + # build response data + data = [] + + for flow in flows: + data.append({ + 'id' : str(flow.id), + 'title' : flow.title, + 'time_created' : str(flow.time_created) + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +### ------ Begin FlowRun Services ------ ### + + + + +def create_flowrun(request: object=None) -> object: + """ + Creates a new `FlowRun` from the passed + "flow_id" & "site_id" + + Args: + 'request': obejct + + Returns: + HTTP Response object + """ + + # get request data + flow_id = request.data.get('flow_id') + site_id = request.data.get('site_id') + configs = request.data.get('configs', None) + + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # update configs + configs = account.configs if configs is None else configs + + # check site + if not Site.objects.filter(id=site_id, account=account).exists(): + data = {'reason': 'site not found'} + record_api_call(request, data, '404') + return Response(data, status=status.HTTP_404_NOT_FOUND) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flowrun', + action='add', id=flow_id, id_type='flow' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get flow if checks passed + flow = Flow.objects.get(id=flow_id) + + # get site if checks passed + site = Site.objects.get(id=site_id) + + # update usage and meter resource + check_and_increment_resource(account.id, 'flowruns') + + # set flowrun_id + flowrun_id = uuid.uuid4() + + # update nodes + nodes = flow.nodes + for i in range(len(nodes)): + nodes[i]['data']['status'] = 'queued' + nodes[i]['data']['finalized'] = False + nodes[i]['data']['time_started'] = None + nodes[i]['data']['time_completed'] = None + nodes[i]['data']['objects'] = [] + + # updates edges + edges = flow.edges + for i in range(len(edges)): + edges[i]['animated'] = False + edges[i]['style'] = None + + # create init log + logs = [{ + 'timestamp': timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f'), + 'message': f'system starting up for run_id: {str(flowrun_id)}', + 'step': '1' + },] + + # create flowrun + flowrun = FlowRun.objects.create( + id = flowrun_id, + flow = flow, + user = flow.user, + account = flow.account, + site = site, + title = flow.title, + nodes = nodes, + edges = edges, + logs = logs, + configs = configs + ) + + # update flow with time_last_run + flow = Flow.objects.get(id=flow_id) + flow.time_last_run = timezone.now() + flow.save() + + # signals.py should pick up this `create()` + # event and then run the first instance of flowr.py + + # serialize and return + data = { + 'id': str(flowrun.id), + 'title': str(flowrun.title), + 'site': str(site.id), + 'time_created': str(flowrun.time_created) + } + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response + + + + +def get_flowruns(request: object=None) -> object: + """ + Get one or more `FlowRun`. + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + flowrun_id = request.query_params.get('flowrun_id') + site_id = request.query_params.get('site_id') + lean = request.query_params.get('lean') -def delete_automation(request, id): + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account - try: - automation = Automation.objects.get(id=id) - except: - data = {'reason': 'cannot find a Automation with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - if automation.account != account: - data = {'reason': 'delete an automation you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # defaults + id = site_id if site_id else flowrun_id + id_type = 'site' if site_id else 'flowrun' - automation.delete() + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flowrun', + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - data = {'message': 'Automation has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) + # get single flowrun + if flowrun_id: + + # get flowrun + flowrun = FlowRun.objects.get(id=flowrun_id) + + # serialize and return + serialized = FlowRunSerializer(flowrun, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # getting site scoped flowruns + if site_id: + flowruns = FlowRun.objects.filter( + site__id=site_id, + account=account + ).order_by('-time_created') + + # get flowruns scoped to account + if not site_id: + flowruns = FlowRun.objects.filter( + account=account + ).order_by('-time_created') + + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(flowruns, request) + serialized = FlowRunSerializer(result_page, many=True, context={'request': request}) + if str(lean).lower() == 'true': + serialized = SmallFlowRunSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') return response +def get_flowrun(request: object=None, id: str=None) -> object: + """ + Get single `FlowRun` from the passed "id" + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='flowrun', + action='get', id=id, id_type='flowrun' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + # get flowruns if checks passed + flowruns = FlowRun.objects.get(id=id) + + # serialize and return + serialized = FlowRunSerializer(flowruns, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) -def create_or_update_report(request): - user = request.user - account = Member.objects.get(user=user).account - report_id = request.data.get('report_id', None) - site_id = request.data.get('site_id', None) - report_type = request.data.get('type', ['lighthouse', 'yellowlab']) - text_color = request.data.get('text_color', '#24262d') - background_color = request.data.get('background_color', '#e1effd') - highlight_color = request.data.get('highlight_color', '#4283f8') - site = Site.objects.get(id=site_id) - info = { - "text_color": text_color, - "background_color": background_color, - "highlight_color": highlight_color, - } +def delete_flowrun(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `FlowRun` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'account' : object - if report_id: - try: - report = Report.objects.get(id=report_id) - except: - data = {'reason': 'cannot find a Report with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + Returns: + HTTP Response object + """ - if report.account != account: - data = {'reason': 'update a Report you do not own'} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account - else: - report = Report.objects.create( - user=request.user, site=site, - account=account - ) + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='flowrun', + action='delete', id=id, id_type='flowrun' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get flowrun if checks passed + flowrun = FlowRun.objects.get(id=id) + + # delete flowrun + flowrun.delete() + + # return response + data = {'message': 'FlowRun deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data - # update report data - report.info = info - report.type = report_type - report.save() - un_cached_report = Report.objects.get(id=report.id) - # generate report - updated_report = R(report=un_cached_report).make_test_report() +def get_flowruns_zapier(request: object=None) -> object: + """ + Get all `FlowRuns` associated with user's Account. - serializer_context = {'request': request,} - data = ReportSerializer(updated_report, context=serializer_context).data - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + _status = request.query_params.get('status') + member = Member.objects.get(user=request.user) + account = member.account + flowruns = None + + # deciding on scope + resource = 'flowrun' + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='flowrun', + action='get', + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + return Response(data, status=check_data['status']) + + # get all account assocoiated flowruns + if flowruns is None: + flowruns = FlowRun.objects.filter( + account=account, + ).exclude( + time_completed=None, + ).order_by('-time_created') + + # filter by _status if requested + if _status is not None: + flowruns = flowruns.filter(status=_status) + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + flowruns = flowruns.filter(site__id__in=id_list).order_by('-time_created') + + # build response data + data = [] + + for run in flowruns: + data.append({ + 'id' : str(run.id), + 'flow' : str(run.flow.id), + 'site' : str(run.site.id), + 'title' : str(run.title), + 'time_created' : str(run.time_created), + 'time_completed' : str(run.time_completed), + 'status' : str(run.status) + }) + + # serialize and return + response = Response(data, status=status.HTTP_200_OK) return response +### ------ Begin Secret Services ------ ### -def get_reports(request): - site_id = request.query_params.get('site_id', None) - report_id = request.query_params.get('report_id', None) - user = request.user - account = Member.objects.get(user=user).account - if site_id: - try: - site = Site.objects.get(id=site_id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - reports = Report.objects.filter(site=site, account=account).order_by('-time_created') - if report_id: - try: - report = Report.objects.get(id=report_id) - except: - data = {'reason': 'cannot find a Report with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - if site_id is None and report_id is None: - reports = Report.objects.filter(user=request.user).order_by('-time_created') +def create_or_update_secret(request: object=None) -> object: + """ + Creates or Updates a `Secret` - paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(reports, request) - serializer_context = {'request': request,} - serialized = ReportSerializer(result_page, many=True, context=serializer_context) - response = paginator.get_paginated_response(serialized.data) - record_api_call(request, response.data, '200') - return response + Args: + 'request': object + Returns: + HTTP Response object + """ - - - -def delete_report(request, id): + # get request data + secret_id = request.data.get('secret_id') + name = request.data.get('name') + value = request.data.get('value') + action = 'update' if secret_id else 'add' + + # get user & account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='secret', + action=action, id=secret_id, id_type='secret' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # encrypt value if passed + f = Fernet(settings.SECRETS_KEY) + bytes_value = bytes(value, 'utf-8') + encrypted_value = f.encrypt(bytes_value).decode('utf-8') + + # update secret + if secret_id: + + # get secret + secret = Secret.objects.get(id=secret_id) - try: - report = Report.objects.get(id=id) - except: - data = {'reason': 'cannot find a Report with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # save new value + secret.value = encrypted_value + secret.save() + + # create new secret + if not secret_id: + secret = Secret.objects.create( + account=account, + user=user, + name=name, + value=encrypted_value + ) - if report.account != account: - data = {'reason': 'delete Reports you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # serialize and return + serialized = SecretSerializer(secret, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) - # remove s3 objects - delete_report_s3_bg.delay(report_id=id) - - # remove report - report.delete() - data = {'message': 'Report has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response +def get_secrets(request: object=None) -> object: + """ + Get one or more `Secrets`. + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + secret_id = request.query_params.get('secret_id') + lean = request.query_params.get('lean') -def get_processes(request): - site_id = request.query_params.get('site_id', None) - process_id = request.query_params.get('process_id', None) + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account - if site_id: - try: - site = Site.objects.get(id=site_id) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - processes = Process.objects.filter(site=site).order_by('-time_created') + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='secret', + action='get', id=secret_id, id_type='secret' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - if process_id: - try: - process = Process.objects.get(id=process_id) - serializer_context = {'request': request,} - data = ProcessSerializer(process, context=serializer_context).data - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response - except: - data = {'reason': 'cannot find a Process with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # get single secret + if secret_id: - if site_id is None and report_id is None: - processes = Process.objects.all().order_by('-time_created') + # get secret + secret = Secret.objects.get(id=secret_id) + + # serialize and return + serialized = SecretSerializer(secret, context={'request': request}) + data = serialized.data + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get secrets scoped to account + secrets = Secret.objects.filter(account=account).order_by('-time_created') + # serialize and return paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(processes, request) - serializer_context = {'request': request,} - serialized = ProcessSerializer(result_page, many=True, context=serializer_context) + result_page = paginator.paginate_queryset(secrets, request) + serialized = SecretSerializer(result_page, many=True, context={'request': request}) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response @@ -1481,551 +6977,1270 @@ def get_processes(request): +def get_secret(request: object=None, id: str=None) -> object: + """ + Get single `Secret` from the passed "id" + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='secret', + action='get', id=id, id_type='secret' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + # get secrets if checks passed + secrets = Secret.objects.get(id=id) + + # serialize and return + serialized = SecretSerializer(secrets, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) +def get_secrets_all(request: object=None) -> object: + """ + Get all `Secrets` associated with the + equesting user's `Account`. -def create_or_update_case(request): - case_id = request.data.get('case_id') - steps = request.data.get('steps') - name = request.data.get('name') - tags = request.data.get('tags') + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='secret', + action='get' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get secrets scoped to account + secrets = Secret.objects.filter(account=account).order_by('-time_created') + + # build into list + data = [] + for secret in secrets: + data.append({ + 'name': secret.name, + 'value': secret.name, + 'task': 'any' + }) + + # return list + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) - if case_id: - try: - case = Case.objects.get(id=case_id) - except: - data = {'reason': 'cannot find a Case with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - if case.account != account: - data = {'reason': 'retrieve Cases you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - else: - case.steps = steps - case.name = name - case.tags = tags - case.save() + + + +def delete_secret(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Secret` associated with the passed "id" + + Args: + 'request' : object, + 'id' : str, + 'user' : object - else: - case = Case.objects.create( - user = request.user, - name = name, - tags = tags, - steps = steps, - account = account - ) + Returns: + HTTP Response object + """ + # get user and account info + if request: + user = request.user + member = Member.objects.get(user=user) + account = member.account - serializer_context = {'request': request,} - data = CaseSerializer(case, context=serializer_context).data - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) - return response + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='secret', + action='delete', id=id, id_type='secret' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get secret if checks passed + secret = Secret.objects.get(id=id) + + # delete secret + secret.delete() + + # return response + data = {'message': 'Secret deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data -def get_cases(request): - case_id = request.query_params.get('case_id') - user = request.user - account = Member.objects.get(user=user).account +### ------ Begin Secret Services ------ ### - if case_id != None: - try: - case = Case.objects.get(id=case_id) - except: - data = {'reason': 'cannot find a Case with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - if case.account != account: - data = {'reason': 'retrieve an Case you do not own',} - return Response(data, status=status.HTTP_403_FORBIDDEN) + + +def create_or_update_chat(request: object=None) -> object: + """ + Creates or Updates a `Chat` + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + chat_id = request.data.get('chat_id') + messages = request.data.get('messages') + _status = request.data.get('value') + action = 'update' if chat_id else 'add' + + # get user & account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='chat', + action=action, id=chat_id, id_type='chat' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # update chat + if chat_id: - serializer_context = {'request': request,} - serialized = CaseSerializer(case, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + # get chat + chat = Chat.objects.get(id=chat_id) + + # update with new values + if messages: + chat.messages = messages + if _status: + chat.status = _status + chat.save() - cases = Case.objects.filter(account=account).order_by('-time_created') - paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(cases, request) - serializer_context = {'request': request,} - serialized = CaseSerializer(result_page, many=True, context=serializer_context) - response = paginator.get_paginated_response(serialized.data) - record_api_call(request, response.data, '200') - return response + # create new chat + if not chat_id: + chat = Chat.objects.create( + account=account, + user=user, + status='active', + messages=messages if messages else [] + ) + # serialize and return + serialized = ChatSerializer(chat, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) -def search_cases(request): - user = request.user - account = Member.objects.get(user=user).account - query = request.query_params.get('query') - cases = Case.objects.filter(account=account, name__icontains=query).order_by('-time_created') - paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(cases, request) - serializer_context = {'request': request,} - serialized = CaseSerializer(result_page, many=True, context=serializer_context) - response = paginator.get_paginated_response(serialized.data) - record_api_call(request, response.data, '200') - return response +def get_chats(request: object=None) -> object: + """ + Get one or more `Chats`. + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + chat_id = request.query_params.get('chat_id') + _status = request.query_params.get('status', 'active') -def delete_case(request, id): + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) - try: - case = Case.objects.get(id=id) - except: - data = {'reason': 'cannot find a Case with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - if case.account != account: - data = {'reason': 'delete an Case you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='chat', + action='get', id=chat_id, id_type='chat' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - case.delete() + # get single chat + if chat_id: - data = {'message': 'Case has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # get chat + chat = Chat.objects.get(id=chat_id) + # serialize and return + serialized = ChatSerializer(chat, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(data, status=status.HTTP_200_OK) + + # get chats scoped to user & status + chats = Chat.objects.filter(user=user, status=_status).order_by('-time_created') + # serialize and return + paginator = LimitOffsetPagination() + result_page = paginator.paginate_queryset(chats, request) + serialized = ChatSerializer(result_page, many=True, context={'request': request}) + response = paginator.get_paginated_response(serialized.data) + record_api_call(request, response.data, '200') + return response +def get_chat(request: object=None, id: str=None) -> object: + """ + Get single `Chat` from the passed "id" + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ -def create_testcase(request, delay=False): - case_id = request.data.get('case_id') - site_id = request.data.get('site_id') - updates = request.data.get('updates') - configs = request.data.get('configs') + # get user and account user = request.user - account = Member.objects.get(user=user).account - + member = Member.objects.get(user=user) - account_is_active = check_account(request) - if not account_is_active: - data = {'reason': 'account not funded',} - record_api_call(request, data, '402') - return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='chat', + action='get', id=id, id_type='chat' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - if case_id and site_id: - try: - case = Case.objects.get(id=case_id, account=account) - except: - data = {'reason': 'cannot find a Case with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # get secrets if checks passed + chat = Chat.objects.get(id=id) + + # serialize and return + serialized = ChatSerializer(chat, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) - try: - site = Site.objects.get(id=site_id, account=account) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - else: - data = {'reason': 'you must provide both site_id and case_id'} - record_api_call(request, data, '409') - return Response(data, status=status.HTTP_409_CONFLICT) - steps = case.steps - for step in steps: - if step['action']['type'] != None: - step['action']['time_created'] = None - step['action']['time_completed'] = None - step['action']['exception'] = None - step['action']['passed'] = None - if step['assertion']['type'] != None: - step['assertion']['time_created'] = None - step['assertion']['time_completed'] = None - step['assertion']['exception'] = None - step['assertion']['passed'] = None - if updates != None: - for update in updates: - steps[int(update['index'])]['action']['value'] = update['value'] +def delete_chat(request: object=None, id: str=None, user: object=None) -> object: + """ + Deletes the `Chat` associated with the passed "id" - if configs is None: - configs = { - 'window_size': '1920,1080', - 'device': 'desktop', - 'interval': 5, - 'min_wait_time': 10, - 'max_wait_time': 30, - } - - testcase = Testcase.objects.create( - case = case, - case_name = case.name, - site = site, - user = request.user, - configs = configs, - steps = steps, - account = account + Args: + 'request' : object, + 'id' : str, + 'user' : object + + Returns: + HTTP Response object + """ + + # get user and account info + user = request.user if request else user + member = Member.objects.get(user=user) + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='chat', + action='delete', id=id, id_type='chat' ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + if request: + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + return data + + # get chat if checks passed + chat = Chat.objects.get(id=id) + + # delete secret + chat.delete() + + # return response + data = {'message': 'Chat deleted'} + if request: + record_api_call(request, data, '200') + response = Response(data, status=status.HTTP_200_OK) + return response + return data + - if delay: - # pass the newly created Testcase to the backgroud task to run - create_testcase_bg.delay(testcase_id=testcase.id) - else: - # running testcase - asyncio.run( - Caser(testcase=testcase).run() - ) - testcase = Testcase.objects.get(id=testcase.id) - serializer_context = {'request': request,} - data = TestcaseSerializer(testcase, context=serializer_context).data - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) - return response + +### ------ Begin Process Services ------ ### -def get_testcases(request): - testcase_id = request.query_params.get('testcase_id') +def get_processes(request: object=None) -> object: + """ + Get one or more `Processes`. + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data site_id = request.query_params.get('site_id') - lean = request.query_params.get('lean') + process_id = request.query_params.get('process_id') + _type = request.query_params.get('type') + object_id = request.query_params.get('object_id') + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account - if testcase_id != None: - try: - testcase = Testcase.objects.get(id=testcase_id) - except: - data = {'reason': 'cannot find a Testcase with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + id = process_id if process_id else site_id + id_type = 'process' if process_id else 'site' + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='process', + action='get', id=id, id_type=id_type + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - if testcase.account != account: - data = {'reason': 'retrieve an Testcase you do not own',} - return Response(data, status=status.HTTP_403_FORBIDDEN) + # get single process + if process_id: - serializer_context = {'request': request,} - serialized = TestcaseSerializer(testcase, context=serializer_context) - data = serialized.data + # get process + process = Process.objects.get(id=process_id) + + # serialize and return + data = ProcessSerializer(process, context={'request': request}).data record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = Response(data, status=status.HTTP_200_OK) + return response - if site_id != None: - try: - site = Site.objects.get(id=site_id, account=account) - except: - data = {'reason': 'cannot find a Site with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - testcases = Testcase.objects.filter(site=site).order_by('-time_created') - - else: - testcases = Testcase.objects.filter(account=account).order_by('-time_created') + # get processes scoped to site + if site_id: + site = Site.objects.get(id=site_id) + processes = Process.objects.filter(site=site).order_by('-time_created') + # get processes scoped to accout and/or type + if site_id is None and process_id is None: + if _type is None and object_id is None: + processes = Process.objects.filter(account=account).order_by('-time_created') + if _type is not None: + processes = Process.objects.filter(account=account, type=_type).order_by('-time_created') + if object_id is not None: + processes = Process.objects.filter(account=account, object_id=object_id).order_by('-time_created') + + # filter out all non permissioned sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + processes = processes.filter(site__id__in=id_list).order_by('-time_created') + + # serialize and return paginator = LimitOffsetPagination() - result_page = paginator.paginate_queryset(testcases, request) - serializer_context = {'request': request,} - serialized = TestcaseSerializer(result_page, many=True, context=serializer_context) - if lean is not None: - serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context) + result_page = paginator.paginate_queryset(processes, request) + serialized = ProcessSerializer(result_page, many=True, context={'request': request}) response = paginator.get_paginated_response(serialized.data) record_api_call(request, response.data, '200') return response -def delete_testcase(request, id): + +def get_process(request: object=None, id: str=None) -> object: + """ + Get single `Process` from the passed "id" + + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + + # get user and account user = request.user - account = Member.objects.get(user=user).account + member = Member.objects.get(user=user) + account = member.account - try: - testcase = Testcase.objects.get(id=id) - except: - data = {'reason': 'cannot find a Testcase with that id'} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='process', + action='get', id=id, id_type='process' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get process if checks passed + process = Process.objects.get(id=id) + + # serialize and return + serialized = ProcessSerializer(process, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) + + + + +def delete_process(request: object=None, id: str=None) -> object: + """ + Get single `Process` from the passed "id" + + Args: + 'request' : object, + 'id' : str - if testcase.account != account: - data = {'reason': 'delete an Testcase you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) + Returns: + HTTP Response object + """ - # remove s3 objects - delete_testcase_s3_bg.delay(testcase_id=id) + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account - testcase.delete() + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='process', + action='delete', id=id, id_type='process' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) - data = {'message': 'Testcase has been deleted',} - record_api_call(request, data, '200') - response = Response(data, status=status.HTTP_200_OK) - return response + # get process if checks passed + process = Process.objects.get(id=id) + # try to revoke celery task + try: + celery.app.control.revoke( + process.info.get('task_id'), + terminate=True, + signal='SIGKILL' + ) + except Exception as e: + print(e) + # delete process + process.delete() + + # return response + data = {'message': 'Process deleted'} + record_api_call(request, data, '200') + return Response(data, status=status.HTTP_200_OK) +### ------ Begin Log Services ------ ### +def get_logs(request: object=None) -> object: + """ + Get one or more `CaseRun`. -def get_logs(request): + Args: + 'request': object + + Returns: + HTTP Response object + """ + # get request data log_id = request.query_params.get('log_id') - request_status = request.query_params.get('status') + request_status = request.query_params.get('success') request_type = request.query_params.get('request_type') - if log_id != None: + # get user + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='log', + action='get', id=log_id, id_type='log' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get single log + if log_id: + + # get log log = Log.objects.get(id=log_id) - if log.user != request.user: - data = {'reason': 'retrieve Logs you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - serializer_context = {'request': request,} - serialized = LogSerializer(log, context=serializer_context) + # serialize and return + serialized = LogSerializer(log, context={'request': request}) data = serialized.data record_api_call(request, data, '200') return Response(data, status=status.HTTP_200_OK) + # filtering logs by passed params if request_status != None and request_type != None: - logs = Log.objects.filter(status=request_status, request_type=request_type, user=request.user).order_by('-time_created') + logs = Log.objects.filter(status=request_status, request_type=request_type, user=user).order_by('-time_created') elif request_status == None and request_type != None: - logs = Log.objects.filter(request_type=request_type, user=request.user).order_by('-time_created') + logs = Log.objects.filter(request_type=request_type, user=user).order_by('-time_created') elif request_status != None and request_type == None: - logs = Log.objects.filter(status=request_status, user=request.user).order_by('-time_created') + logs = Log.objects.filter(status=request_status, user=user).order_by('-time_created') else: - logs = Log.objects.filter(user=request.user).order_by('-time_created') + logs = Log.objects.filter(user=user).order_by('-time_created') + # serialize and return paginator = LimitOffsetPagination() result_page = paginator.paginate_queryset(logs, request) - serializer_context = {'request': request,} - serialized = LogSerializer(result_page, many=True, context=serializer_context) + serialized = LogSerializer(result_page, many=True, context={'request': request}) response = paginator.get_paginated_response(serialized.data) return response +def get_log(request: object=None, id: str=None) -> object: + """ + Get single `Log` from the passed "id" + Args: + 'request' : object, + 'id' : str + + Returns: + HTTP Response object + """ + # get user and account + user = request.user + member = Member.objects.get(user=user) + account = member.account - -def migrate_site(request, delay=False): - login_url = request.data.get('login_url', None) - admin_url = request.data.get('admin_url', None) - plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator') - username = request.data.get('username', None) - password = request.data.get('password', None) - site_id = request.data.get('site_id', None) - email_address = request.data.get('email_address', None) - destination_url = request.data.get('destination_url', None) - sftp_address = request.data.get('sftp_address', None) - dbname = request.data.get('dbname', None) - sftp_username = request.data.get('sftp_username', None) - sftp_password = request.data.get('sftp_password', None) - wait_time = request.data.get('wait_time', 30) - driver = request.data.get('driver', 'puppeteer') - - site = Site.objects.get(id=site_id) - process = Process.objects.create( - site=site, - type='migration' + # check account and resource + check_data = check_permissions_and_usage( + member=member, resource='log', + action='get', id=id, id_type='log' ) - process_id = process.id - - if delay: - migrate_site_bg.delay( - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - plugin_name, - wait_time, - process_id, - driver - ) - - serializer_context = {'request': request,} - data = ProcessSerializer(process, context=serializer_context).data - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) - return response - + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + # get log if checks passed + log = Log.objects.get(id=id) + + # serialize and return + serialized = LogSerializer(log, context={'request': request}) + record_api_call(request, serialized.data, '200') + return Response(serialized.data, status=status.HTTP_200_OK) - if driver == 'selenium': - # init wordpress - wp = W( - login_url=login_url, - admin_url=admin_url, - username=username, - password=password, - email_address=email_address, - destination_url=destination_url, - sftp_address=sftp_address, - dbname=dbname, - sftp_username=sftp_username, - sftp_password=sftp_password, - wait_time=wait_time, - process_id=process.id - ) - # login - wp_status = wp.login() - # adjust lang - wp_status = wp.begin_lang_check() +### ------ Begin Search Services ------ ### - # install plugin - wp_status = wp.install_plugin(plugin_name=plugin_name) - # launch migration - wp_status = wp.launch_migration() - # run migration - wp_status = wp.run_migration() - # re adjust lang - # wp_status = wp.end_lang_check() +def search_resources(request: object=None) -> object: + """ + This method will search for any `Page` or `Site` + that is associated with the user's `Account` and + matches the query string. - if wp_status: - data = { - 'status': 'success', - 'message': 'site migration succeeded' - } - else: - data = { - 'status': 'failed', - 'message': 'site migration failed' + Expects: + 'query': the query string + + Returns: + data -> [ + { + 'str' : , + 'type': , + 'path': , + 'id' : , } + ... + ] + """ - response = Response(data, status=status.HTTP_200_OK) - record_api_call(request, data, '200') - return response + # get data + query = request.query_params.get('query') + user = request.user + member = Member.objects.get(user=user) + account = member.account + actions = member.permissions.get('actions', []) + resources = member.permissions.get('resources', []) + allowed_ids = [item['id'] for item in member.permissions.get('sites')] + data = [] + cases = [] + pages = [] + sites = [] + issues = [] + flows = [] + + # check action permissons + if 'get' not in actions: + data = {'reason': 'not allowed'} + record_api_call(request, data, '403') + return Response(data, status=status.HTTP_403_FORBIDDEN) - else: + # check for object specification i.e 'site:', 'case:', 'issue:' + resource_type = query.replace('https://', '').replace('http://', '').split(':')[0] + query = query.replace('https://', '').replace('http://', '').split(':')[-1] - # init wordpress for puppeteer - wp_status = asyncio.run( - W_P( - login_url=login_url, - admin_url=admin_url, - username=username, - password=password, - wait_time=wait_time, - ).run_full(plugin_name=plugin_name) + # search for sites + if (resource_type == 'site' or resource_type == query) and 'site' in resources: + sites = Site.objects.filter(account=account).filter( + site_url__icontains=query + ) + # filter out all non permisioned + if len(allowed_ids) > 0: + sites = sites.filter(id__in=allowed_ids) + + # search for pages + if (resource_type == 'page' or resource_type == query) and 'page' in resources: + pages = Page.objects.filter(account=account).filter( + page_url__icontains=query + ) + # filter out all non permisioned + if len(allowed_ids) > 0: + pages = pages.filter(site__id__in=allowed_ids) + + # search for cases + if (resource_type == 'case' or resource_type == query) and 'case' in resources: + cases = Case.objects.filter(account=account).filter( + title__icontains=query + ) + # filter out all non permisioned + if len(allowed_ids) > 0: + cases = cases.filter(site__id__in=allowed_ids) + + # search for issues + if (resource_type == 'issue' or resource_type == query) and 'issue' in resources: + issues = Issue.objects.filter(account=account).filter( + title__icontains=query + ) + # filter out all non permisioned + if len(allowed_ids) > 0: + new_ids = allowed_ids + for id in allowed_ids: + for page in Page.objects.filter(site__id=id): + new_ids.append(str(page.id)) + issues = issues.filter(affected__id__in=new_ids) + + # search for flows + if (resource_type == 'flow' or resource_type == query) and 'flow' in resources: + flows = Flow.objects.filter(account=account).filter( + title__icontains=query ) - if wp_status: - data = { - 'status': 'success', - 'message': 'site migration succeeded' - } - else: - data = { - 'status': 'failed', - 'message': 'site migration failed' - } + # adding first several sites if present + i = 0 + sites_allowed = 10 if resource_type == 'site' else 3 + while i <= sites_allowed and i <= (len(sites)-1): + data.append({ + 'str': str(sites[i].site_url), + 'path': f'/site/{sites[i].id}', + 'id' : str(sites[i].id), + 'type': 'site', + }) + i+=1 + + # adding first several pages if present + i = 0 + max_pages = 10 if resource_type == 'page' else 3 + while i <= max_pages and i <= (len(pages)-1): + data.append({ + 'str': str(pages[i].page_url), + 'path': f'/page/{pages[i].id}', + 'id' : str(pages[i].id), + 'type': 'page', + }) + i+=1 + + # adding first several cases if present + i = 0 + max_cases = 10 if resource_type == 'case' else 3 + while i <= max_cases and i <= (len(cases)-1): + data.append({ + 'str': str(cases[i].title), + 'path': f'/case/{cases[i].id}', + 'id' : str(cases[i].id), + 'type': 'case', + }) + i+=1 + + # adding first several issues if present + i = 0 + max_issues = 10 if resource_type == 'issue' else 3 + while i <= max_issues and i <= (len(issues)-1): + data.append({ + 'str': str(issues[i].title), + 'path': f'/issue/{issues[i].id}', + 'id' : str(issues[i].id), + 'type': 'issue', + }) + i+=1 + + # adding first several flows if present + i = 0 + max_flows = 10 if resource_type == 'flows' else 2 + while i <= max_flows and i <= (len(flows)-1): + data.append({ + 'str': str(flows[i].title), + 'path': f'/flow/{flows[i].id}', + 'id' : str(flows[i].id), + 'type': 'flow', + }) + i+=1 + + # return response + response = Response(data, status=status.HTTP_200_OK) + return response - response = Response(data, status=status.HTTP_200_OK) - record_api_call(request, data, '200') - return response +def get_tags(request: object=None) -> object: + """ + Retrieves a list of all "Tags" assciated with any object in + user's account + + Expects: + None + + Returns: + HTTP Response object + """ + + # getting account + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # default + tags = [] + pages = [] + all = [] + # get all site & pages + sites = Site.objects.filter(account=account) + # filter out all non permissioned sites + if len(member.permissions.get('sites', [])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + sites = sites.filter(id__in=id_list) + # get all pages + for site in sites: + pages += Page.objects.filter(site=site) + + # group as one -- super slow!!! fix at somepoint + all = list(sites) + pages + for i in all: + if i.tags: + for tag in i.tags: + tags.append(tag) + + # format data + data = { + 'tags': list(set(tags)) + } + # return response + response = Response(data, status=status.HTTP_200_OK) + return response -def create_site_screenshot(request): - user = request.user - site_id = request.data.get('site_id', None) - url = request.data.get('url', None) - configs = request.data.get('configs', None) - site = None - if site_id is not None: - site = Site.objects.get(id=site_id) +def get_devices(request: object=None) -> object: + """ + Retrieves a list of all Cursion "devices" + + Expects: None - if configs is not None: - if configs['driver'] == 'puppeteer': - data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs)) - elif configs['driver'] == 'selenium': - data = I().screenshot(site=site, url=url, configs=configs) - else: - data = I().screenshot(site=site, url=url, configs=configs) - record_api_call(request, data, '201') - response = Response(data, status=status.HTTP_201_CREATED) + Returns: + HTTP Response object + """ + + # format data + data = { + 'devices': devices + } + + # return response + response = Response(data, status=status.HTTP_200_OK) return response +### ------ Begin Metrics Services ------ ### -def get_home_stats(request): + +def get_home_metrics(request: object=None) -> object: + """ + Builds metrics for account "Home" view + on Cursion.client + + Args: + 'request' : object + + Returns: + HTTP Response object + """ + + # get user, account, sites, & issues user = request.user - account = Member.objects.get(user=user).account - sites = Site.objects.filter(account=account) - site_count = sites.count() - test_count = 0 - scan_count = 0 - schedule_count = 0 - for site in sites: - tests = Test.objects.filter(site=site) - scans = Scan.objects.filter(site=site) - schedules = Schedule.objects.filter(site=site) - test_count = test_count + tests.count() - scan_count = scan_count + scans.count() - schedule_count = schedule_count + schedules.count() + member = Member.objects.get(user=user) + account = member.account + sites = Site.objects.filter(account=account).count() + issues = Issue.objects.filter(account=account, status='open') + schedules = Schedule.objects.filter(account=account).count() + + # filter issues by allowed sites + if len(member.permissions.get('sites',[])) != 0: + id_list = [item['id'] for item in member.permissions.get('sites')] + new_ids = id_list + for id in id_list: + for page in Page.objects.filter(site__id=id): + new_ids.append(str(page.id)) + issues = issues.filter(affected__id__in=new_ids) + + # setting resource defaults + tests = account.usage['tests'] + scans = account.usage['scans'] + caseruns = account.usage['caseruns'] + flowruns = account.usage.get('flowruns', 0) + issues = issues.count() + + # calculate usages + sites_usage = round((sites/account.usage['sites_allowed'])*100, 2) if sites > 0 else 0 + schedules_usage = round((schedules/account.usage['schedules_allowed'])*100, 2) if schedules > 0 else 0 + scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0 + tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0 + caseruns_usage = round((caseruns/account.usage['caseruns_allowed'])*100, 2) if caseruns > 0 else 0 + flowruns_usage = round((flowruns/account.usage['flowruns_allowed'])*100, 2) if flowruns > 0 else 0 + + # format data + data = { + "sites": sites, + "sites_usage": sites_usage, + "tests": tests, + "tests_usage": tests_usage, + "scans": scans, + "scans_usage": scans_usage, + "schedules": schedules, + "schedules_usage": schedules_usage, + "caseruns": caseruns, + "caseruns_usage": caseruns_usage, + "flowruns": flowruns, + "flowruns_usage": flowruns_usage, + "open_issues": issues, + } + + # return response + response = Response(data, status=status.HTTP_200_OK) + return response + + + + +def get_site_metrics(request: object=None) -> object: + """ + Builds metrics for account "Site" view + on Cursion.client + + Args: + 'request' : object + + Returns: + HTTP Response object + """ + + # get user, account, site, & pages + user = request.user + member = Member.objects.get(user=user) + account = member.account + site_id = request.query_params.get('site_id') + site = Site.objects.get(id=site_id) + sites_allowed = account.usage['sites_allowed'] + pages = Page.objects.filter(site=site) + + # get last reset day + last_usage_date_str = account.meta.get('last_usage_reset') + last_usage_date = None + if last_usage_date_str: + last_usage_date = datetime.fromisoformat(last_usage_date_str.replace('Z', '')) + else: + last_usage_date = timezone.now() - timedelta(30) + + # get scans + scans = Scan.objects.filter( + site=site, + time_created__gte=last_usage_date + ).count() + + # get tests + tests = Test.objects.filter( + site=site, + time_created__gte=last_usage_date + ).count() + + # get caseruns + caseruns = CaseRun.objects.filter( + site=site, + time_created__gte=last_usage_date + ).count() + # get flowruns + flowruns = FlowRun.objects.filter( + site=site, + time_created__gte=last_usage_date + ).count() + + # get site scoped schedules + schedules = Schedule.objects.filter( + resources__icontains=str(site.id), scope='site', + account=account + ).count() + + # calculating page scoped schedules + for page in pages: + schedules += Schedule.objects.filter( + resources__icontains=str(page.id), scope='page', + account=account + ).count() + + # calculate usage + pages = pages.count() + pages_usage = round((pages/account.usage['pages_allowed'])*100, 2) if pages > 0 else 0 + schedules_usage = round((schedules/account.usage['schedules_allowed'])*100, 2) if schedules > 0 else 0 + scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0 + tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0 + caseruns_usage = round((caseruns/account.usage['caseruns_allowed'])*100, 2) if caseruns > 0 else 0 + flowruns_usage = round((flowruns/account.usage['flowruns_allowed'])*100, 2) if flowruns > 0 else 0 + + # format data data = { - "sites": site_count, - "tests": test_count, - "scans": scan_count, - "schedules": schedule_count, + "pages": pages, + "pages_usage": pages_usage, + "tests": tests, + "tests_usage": tests_usage, + "scans": scans, + "scans_usage": scans_usage, + "schedules": schedules, + "schedules_usage": schedules_usage, + "caseruns": caseruns, + "caseruns_usage": caseruns_usage, + "flowruns": flowruns, + "flowruns_usage": flowruns_usage, } + + # return response response = Response(data, status=status.HTTP_200_OK) return response + + +def get_page_metrics(request: object=None) -> object: + """ + Builds `Scan` and `Test` metrics for + "Page" view on Cursion.client + + Args: + 'request' : object + + Returns: + HTTP Response object + """ + + # get user, account, member + user = request.user + member = Member.objects.get(user=user) + account = member.account + + # request data + page_id = request.query_params.get('page_id') + weeks = request.query_params.get('weeks') + + # checking account and resource + check_data = check_permissions_and_usage( + member=member, resource='page', + action='get', id=page_id, id_type='page' + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get datetime, x-weeks ago + timeago = timezone.now() - timedelta(weeks=int(weeks)) + + # get scans + scans_raw = ( + Scan.objects.filter(page_id=page_id, time_completed__gte=timeago) + .exclude(score=None) + .order_by('-time_created') + ) + scans = [ + { + 'id' : str(s.id), + 'score' : s.score, + 'time_created' : s.time_created + } + for s in scans_raw + ] + + # get tests + tests_raw = ( + Test.objects.filter(page_id=page_id, time_completed__gte=timeago) + .exclude(status='incomplete') + .exclude(post_scan=None) + .order_by('-time_created') + ) + tests = [ + { + 'id' : str(t.id), + 'score' : t.score, + 'health' : t.post_scan.score, + 'time_created' : t.time_created + } + for t in tests_raw + ] + + # return respose + return Response({'scans': scans, 'tests': tests}, status.HTTP_200_OK) + + + + +def get_celery_metrics(request: object=None) -> object: + """ + Builds metrics for current Celery task load. + Used to provision and terminate new pods in + k8s cluster on PROD + + Args: + 'request' : object + + Returns: + HTTP Response object + """ + + cached = cache.get("celery_metrics") + if cached: + return Response(cached, status=status.HTTP_200_OK) + + queue_param = None + try: + queue_param = request.query_params.get('queue') + except Exception: + queue_param = None + + scheduled_queue = getattr(settings, 'CELERY_QUEUE_SCHEDULED', 'scheduled') + on_demand_queue = getattr(settings, 'CELERY_QUEUE_ON_DEMAND', 'on_demand') + + try: + redis_client = Redis.from_url( + settings.CELERY_BROKER_URL, + socket_connect_timeout=2 + ) + if queue_param in {scheduled_queue, on_demand_queue, 'celery'}: + redis_queue_len = redis_client.llen(queue_param) + scheduled_len = redis_client.llen(scheduled_queue) if queue_param == scheduled_queue else 0 + on_demand_len = redis_client.llen(on_demand_queue) if queue_param == on_demand_queue else 0 + else: + # default to total queued across known queues + scheduled_len = redis_client.llen(scheduled_queue) + on_demand_len = redis_client.llen(on_demand_queue) + legacy_len = redis_client.llen('celery') + redis_queue_len = scheduled_len + on_demand_len + legacy_len + except RedisError: + redis_queue_len = 0 + scheduled_len = 0 + on_demand_len = 0 + + try: + i = celery.app.control.inspect() + reserved = i.reserved() or {} + active = i.active() or {} + except Exception: + reserved, active = {}, {} + + def _count_tasks_by_queue(tasks_by_worker: dict, queue_name: str | None) -> int: + count = 0 + for tasks in (tasks_by_worker or {}).values(): + for task in tasks or []: + if not queue_name: + count += 1 + continue + delivery = task.get('delivery_info') or {} + routing_key = delivery.get('routing_key') + if routing_key == queue_name: + count += 1 + return count + + filter_queue = queue_param if queue_param in {scheduled_queue, on_demand_queue, 'celery'} else None + + # calc tasks + num_tasks = _count_tasks_by_queue(reserved, filter_queue) + _count_tasks_by_queue(active, filter_queue) + + # calc replicas + num_replicas = len(reserved) + + # calc ratio & working_len + ratio = num_tasks / num_replicas if num_replicas else 0 + working_len = redis_queue_len + num_tasks + + data = { + "num_tasks": num_tasks, + "num_replicas": num_replicas, + "ratio": ratio, + "redis_queue": redis_queue_len, + "redis_queue_scheduled": scheduled_len, + "redis_queue_on_demand": on_demand_len, + "working_len": working_len + } + + cache.set("celery_metrics", data, timeout=10) + return Response(data, status=status.HTTP_200_OK) + + + + +### ------ Begin Beta Services ------ ### + + + + +def migrate_site(request: object=None) -> object: + """ + Initiate a `Site` migration task in background + + Args: + 'request': object + + Returns: + HTTP Response object + """ + + # get request data + login_url = request.data.get('login_url', None) + admin_url = request.data.get('admin_url', None) + plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator') + username = request.data.get('username', None) + password = request.data.get('password', None) + site_id = request.data.get('site_id', None) + email_address = request.data.get('email_address', None) + destination_url = request.data.get('destination_url', None) + sftp_address = request.data.get('sftp_address', None) + dbname = request.data.get('dbname', None) + sftp_username = request.data.get('sftp_username', None) + sftp_password = request.data.get('sftp_password', None) + wait_time = request.data.get('wait_time', 30) + driver = request.data.get('driver', 'selenium') + + # checking account and resource + check_data = check_permissions_and_usage( + request=request, resource='site', + site_id=site_id + ) + if not check_data['allowed']: + data = {'reason': check_data['error']} + record_api_call(request, data, check_data['code']) + return Response(data, status=check_data['status']) + + # get site if checks passed + site = Site.objects.get(id=site_id) + + # create new Process + process = Process.objects.create( + site=site, + type='migration' + ) + + # start migrtation task in background + migrate_site_bg.apply_async( + args=[ + login_url, + admin_url, + username, + password, + email_address, + destination_url, + sftp_address, + dbname, + sftp_username, + sftp_password, + plugin_name, + wait_time, + str(process.id), + driver, + ], + queue=ON_DEMAND_QUEUE, + routing_key=ON_DEMAND_QUEUE, + ) + + # serialize and return + data = ProcessSerializer(process, context={'request': request}).data + record_api_call(request, data, '201') + response = Response(data, status=status.HTTP_201_CREATED) + return response diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py deleted file mode 100644 index 8dc20b9e..00000000 --- a/app/api/v1/ops/tasks.py +++ /dev/null @@ -1,360 +0,0 @@ -from ...models import * -from ...utils.scanner import Scanner as S -from ...utils.tester import Tester as T -from ...utils.reporter import Reporter as R -from ...utils.wordpress import Wordpress as W -from ...utils.wordpress_p import Wordpress as W_P -from ...utils.automations import automation -from ...utils.caser import Caser -import boto3, asyncio -from scanerr import settings -from ...utils.scanner import ( - _html_and_logs, _vrt, _lighthouse, - _yellowlab -) - - - -def create_site_task(site_id, scan_id, configs): - site = Site.objects.get(id=site_id) - scan = Scan.objects.get(id=scan_id) - S(site=site, scan=scan, configs=configs).first_scan() - return site - - -def create_scan_task( - scan_id=None, - site_id=None, - type=['full'], - automation_id=None, - configs=None, - tags=None, - ): - if scan_id is not None: - created_scan = Scan.objects.get(id=scan_id) - elif site_id is not None: - site = Site.objects.get(id=site_id) - created_scan = Scan.objects.create( - site=site, - type=type, - configs=configs, - tags=tags, - ) - scan = S(scan=created_scan, configs=configs).first_scan() - if automation_id: - automation(automation_id, scan.id) - return scan - - - - - -def run_html_and_logs_task(scan_id=None): - scan = _html_and_logs(scan_id) - return scan - -def run_vrt_task(scan_id=None): - scan = _vrt(scan_id) - return scan - -def run_lighthouse_task(scan_id=None): - scan = _lighthouse(scan_id) - return scan - -def run_yellowlab_task(scan_id=None): - scan = _yellowlab(scan_id) - return scan - - - - - -def create_test_task( - test_id=None, - site_id=None, - automation_id=None, - configs=None, - type=['full'], - index=None, - pre_scan=None, - post_scan=None, - tags=None, - ): - - if test_id is not None: - created_test = Test.objects.get(id=test_id) - site = created_test.site - elif site_id is not None: - site = Site.objects.get(id=site_id) - created_test = Test.objects.create( - site=site, - type=type, - tags=tags, - ) - - if pre_scan is not None: - pre_scan = Scan.objects.get(id=pre_scan) - if post_scan is not None: - post_scan = Scan.objects.get(id=post_scan) - - if post_scan is None and pre_scan is not None: - post_scan = S(site=site, scan=pre_scan, configs=configs, type=type).second_scan() - - if pre_scan is None and post_scan is None: - new_scan = S(site=site, configs=configs, type=type) - post_scan = new_scan.second_scan() - pre_scan = post_scan.paired_scan - - # updating parired scans - pre_scan.paired_scan = post_scan - post_scan.paried_scan = pre_scan - pre_scan.save() - post_scan.save() - - # updating test object - created_test.type = type - created_test.pre_scan = pre_scan - created_test.post_scan = post_scan - created_test.save() - - - test = T(test=created_test).run_test(index=index) - if automation_id: - automation(automation_id, test.id) - return test - - - - -def create_report_task(site_id, automation_id=None): - site = Site.objects.get(id=site_id) - if Report.objects.filter(site=site).exists(): - report = Report.objects.filter(site=site).order_by('-time_created')[0] - else: - info = { - "text_color": '#24262d', - "background_color": '#e1effd', - "highlight_color": '#ffffff', - } - report = Report.objects.create( - user=site.user, - site=site, - info=info, - type=['lighthouse', 'yellowlab'] - ) - - - report = R(report=report).make_test_report() - if automation_id: - automation(automation_id, report.id) - return report - - - - - - -def delete_site_s3(site_id): - # setup boto3 configurations - s3 = boto3.resource('s3', - aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # deleting s3 objects - try: - bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) - bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete() - except: - pass - - return - - - - -def delete_testcase_s3(testcase_id): - # setup boto3 configurations - s3 = boto3.resource('s3', - aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # deleting s3 objects - try: - bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) - bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete() - except: - pass - - return - - - - - -def delete_report_s3(report_id): - # setup boto3 configurations - s3 = boto3.resource('s3', - aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID), - aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY), - region_name=str(settings.AWS_S3_REGION_NAME), - endpoint_url=str(settings.AWS_S3_ENDPOINT_URL) - ) - - # get site - site = Report.objects.get(id=report_id).site - - # deleting s3 objects - bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) - bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete() - - return - - - - -def create_testcase_task( - testcase_id=None, - site_id=None, - case_id=None, - updates=None, - configs=None, - automation_id=None - ): - - if testcase_id != None: - testcase = Testcase.objects.get(id=testcase_id) - - else: - case = Case.objects.get(id=case_id) - site = Site.objects.get(id=site_id) - steps = case.steps - for step in steps: - if step['action']['type'] != None: - step['action']['time_created'] = None - step['action']['time_completed'] = None - step['action']['exception'] = None - step['action']['passed'] = None - - if step['assertion']['type'] != None: - step['assertion']['time_created'] = None - step['assertion']['time_completed'] = None - step['assertion']['exception'] = None - step['assertion']['passed'] = None - - if updates != None: - for update in updates: - steps[int(update['index'])]['action']['value'] = update['value'] - - if configs is None: - configs = { - 'window_size': '1920,1080', - 'device': 'desktop', - 'interval': 5, - 'min_wait_time': 10, - 'max_wait_time': 30, - } - - testcase = Testcase.objects.create( - case = case, - case_name = case.name, - site = site, - user = site.user, - configs = configs, - steps = steps - ) - - - # running testcase - testresult = asyncio.run( - Caser(testcase=testcase).run() - ) - - if automation_id: - automation(automation_id, testcase.id) - - return - - - - -def migrate_site_task( - login_url, - admin_url, - username, - password, - email_address, - destination_url, - sftp_address, - dbname, - sftp_username, - sftp_password, - plugin_name, - wait_time, - process_id, - driver, - - ): - - if driver == 'selenium': - # init wordpress - wp = W( - login_url=login_url, - admin_url=admin_url, - username=username, - password=password, - email_address=email_address, - destination_url=destination_url, - sftp_address=sftp_address, - dbname=dbname, - sftp_username=sftp_username, - sftp_password=sftp_password, - wait_time=wait_time, - process_id=process_id, - - ) - - # login - wp_status = wp.login() - # adjust lang - wp_status = wp.begin_lang_check() - # install plugin - wp_status = wp.install_plugin(plugin_name=plugin_name) - # launch migration - wp_status = wp.launch_migration() - # run migration - wp_status = wp.run_migration() - # re adjust lang - # wp_status = wp.end_lang_check() - - else: - # init wordpress for puppeteer - wp_status = asyncio.run( - W_P( - login_url=login_url, - admin_url=admin_url, - username=username, - password=password, - email_address=email_address, - destination_url=destination_url, - sftp_address=sftp_address, - dbname=dbname, - sftp_username=sftp_username, - sftp_password=sftp_password, - wait_time=wait_time, - process_id=process_id, - ).run_full(plugin_name=plugin_name) - ) - - return - - - - - - \ No newline at end of file diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py index f0d0f7ab..d9c9057f 100644 --- a/app/api/v1/ops/urls.py +++ b/app/api/v1/ops/urls.py @@ -3,38 +3,81 @@ urlpatterns = [ + path('tasks/retry', views.TasksRetry.as_view(), name='tasks-retry'), path('site', views.Sites.as_view(), name='site'), path('site/', views.SiteDetail.as_view(), name='site-detail'), - path('site/delay', views.SiteDelay.as_view(), name='site-delay'), + path('site//crawl', views.SiteCrawl.as_view(), name='site-crawl'), path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'), + path('sites/zapier', views.SitesZapier.as_view(), name='sites-zapier'), + path('page', views.Pages.as_view(), name='page'), + path('page/', views.PageDetail.as_view(), name='page-detail'), + path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'), + path('pages/zapier', views.PagesZapier.as_view(), name='pages-zapier'), path('scan', views.Scans.as_view(), name='scan'), path('scan/', views.ScanDetail.as_view(), name='scan-detail'), path('scan//lean', views.ScanLean.as_view(), name='scan-lean'), - path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'), path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'), + path('scans/create', views.ScansCreate.as_view(), name='scans-create'), + path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'), path('test', views.Tests.as_view(), name='test'), path('test/', views.TestDetail.as_view(), name='test-detail'), path('test//lean', views.TestLean.as_view(), name='test-lean'), - path('test/delay', views.TestDelay.as_view(), name='test-delay'), path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'), - path('log', views.Logs.as_view(), name='log'), - path('log/', views.LogDetail.as_view(), name='log-detail'), - path('schedule', views.Schedules.as_view(), name='schedule'), - path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), - path('automation', views.Automations.as_view(), name='automation'), - path('automation/', views.AutomationDetail.as_view(), name='automation-detail'), + path('tests/create', views.TestsCreate.as_view(), name='tests-create'), + path('tests/zapier', views.TestsZapier.as_view(), name='tests-zapier'), + path('case', views.Cases.as_view(), name='case'), + path('case/', views.CaseDetail.as_view(), name='case-detail'), + path('case/pre-run', views.CasePreRun.as_view(), name='case-pre-run'), + path('case/search', views.CasesSearch.as_view(), name='case-search'), + path('case/auto', views.AutoCases.as_view(), name='case-auto'), + path('case/copy', views.CopyCases.as_view(), name='case-copy'), + path('cases/delete', views.CasesDelete.as_view(), name='cases-delete'), + path('cases/zapier', views.CasesZapier.as_view(), name='cases-zapier'), + path('caserun', views.CaseRuns.as_view(), name='caserun'), + path('caserun/', views.CaseRunDetail.as_view(), name='caserun-detail'), + path('caseruns/zapier', views.CaseRunsZapier.as_view(), name='caseruns-zapier'), + path('flow', views.Flows.as_view(), name='case'), + path('flow/', views.FlowDetail.as_view(), name='flow-detail'), + path('flow/search', views.FlowsSearch.as_view(), name='flows-search'), + path('flow/copy', views.CopyFlows.as_view(), name='flows-copy'), + path('flows/delete', views.FlowsDelete.as_view(), name='flows-delete'), + path('flows/zapier', views.FlowsZapier.as_view(), name='flows-zapier'), + path('flowrun', views.FlowRuns.as_view(), name='flowruns'), + path('flowrun/', views.FlowRunDetail.as_view(), name='flowruns-detail'), + path('flowruns/zapier', views.FlowRunsZapier.as_view(), name='flowruns-zapier'), + path('issue', views.Issues.as_view(), name='issue'), + path('issue/generate', views.IssueGenerate.as_view(), name='issue-generate'), + path('issue/search', views.IssuesSearch.as_view(), name='issue-search'), + path('issue/', views.IssueDetail.as_view(), name='issue-detail'), + path('issues/update', views.IssuesUpdate.as_view(), name='issues-update'), + path('issues/delete', views.IssuesDelete.as_view(), name='issues-delete'), + path('issues/zapier', views.IssuesZapier.as_view(), name='issues-zapier'), path('report', views.Reports.as_view(), name='report'), path('report/', views.ReportDetail.as_view(), name='report-detail'), - path('home-stats', views.HomeStats.as_view(), name='home-stats'), + path('schedule', views.Schedules.as_view(), name='schedule'), + path('schedule/', views.ScheduleDetail.as_view(), name='schedule-detail'), + path('schedule/run', views.ScheduleRun.as_view(), name='schedule-run'), + path('schedules/update', views.SchedulesUpdate.as_view(), name='schedule-update'), + path('schedules/delete', views.SchedulesDelete.as_view(), name='schedule-delete'), + path('alert', views.Alerts.as_view(), name='alert'), + path('alert/', views.AlertDetail.as_view(), name='alert-detail'), path('process', views.Processes.as_view(), name='process'), path('process/', views.ProcessDetail.as_view(), name='process-detail'), - path('case', views.Cases.as_view(), name='case'), - path('case/', views.CaseDetail.as_view(), name='case-detail'), - path('case/search', views.CasesSearch.as_view(), name='case-search'), - path('testcase', views.Testcases.as_view(), name='testcase'), - path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'), - path('testcase/', views.TestcaseDetail.as_view(), name='testcase-detail'), + path('secret', views.Secrets.as_view(), name='secret'), + path('secret/', views.SecretDetail.as_view(), name='secret-detail'), + path('secrets', views.SecretsAll.as_view(), name='secrets-all'), + path('chat', views.Chats.as_view(), name='chat'), + path('chat/', views.ChatDetail.as_view(), name='chat-detail'), + path('log', views.Logs.as_view(), name='log'), + path('log/', views.LogDetail.as_view(), name='log-detail'), + path('search', views.Search.as_view(), name='search'), + path('device', views.Device.as_view(), name='device'), + path('tag', views.Tag.as_view(), name='tag'), + path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'), + path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'), + path('metrics/page', views.PageMetrics.as_view(), name='page-metrics'), + path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'), path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'), - path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'), - path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), + path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), + path('beta/report/export', views.ExportReport.as_view(), name='export-report'), ] \ No newline at end of file diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py index 2bc92ea0..4f19ad8e 100644 --- a/app/api/v1/ops/views.py +++ b/app/api/v1/ops/views.py @@ -1,55 +1,59 @@ -from django.shortcuts import render -from rest_framework.response import Response -from rest_framework import status -from django.contrib.auth.models import User -from django.shortcuts import get_object_or_404 from ...models import * -from django.urls import path, include -from rest_framework import routers, serializers, viewsets -from rest_framework.viewsets import ViewSet -from rest_framework.permissions import AllowAny +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.views import APIView -from rest_framework.permissions import IsAuthenticated -from django.views.decorators.csrf import ensure_csrf_cookie from rest_framework.pagination import LimitOffsetPagination -from django.urls import resolve from .serializers import * from .services import * -class Sites(APIView): + + + +### ------ Begin Task Views ------ ### + + + + +class TasksRetry(APIView): permission_classes = (AllowAny,) + http_method_names = ['get'] + + def get(self, request): + response = retry_failed_tasks(request) + return response + + + + +### ------ Begin Site Views ------ ### + + + + +class Sites(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post', 'get'] pagination_class = LimitOffsetPagination def post(self, request): - response = create_site(request) + response = create_or_update_site(request) return response def get(self, request): response = get_sites(request) return response - + + class SiteDetail(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete'] def get(self, request, id): - site = get_object_or_404(Site, pk=id) - user = request.user - account = Member.objects.get(user=user).account - if site.account != account: - data = {'reason': 'you cannot retrieve a Site you do not own',} - record_api_call(request, data, '401') - return Response(data, status=status.HTTP_403_FORBIDDEN) - serializer_context = {'request': request,} - serialized = SiteSerializer(site, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_site(request, id) + return response def delete(self, request, id): response = delete_site(request, id) @@ -57,18 +61,20 @@ def delete(self, request, id): -class SiteDelay(APIView): - permission_classes = (AllowAny,) + +class SiteCrawl(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post',] - def post(self, request): - response = create_site(request, delay=True) + def post(self, request, id): + response = crawl_site(request, id) return response + class SitesDelete(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): @@ -78,8 +84,82 @@ def post(self, request): +class SitesZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_sites_zapier(request) + return response + + + + +### ------ Begin Page Views ------ ### + + + + +class Pages(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] + pagination_class = LimitOffsetPagination + + def post(self, request): + response = create_or_update_page(request) + return response + + def get(self, request): + response = get_pages(request) + return response + + + + +class PageDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_page(request, id) + return response + + def delete(self, request, id): + response = delete_page(request, id) + return response + + + + +class PagesDelete(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_pages(request) + return response + + + + +class PagesZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_pages_zapier(request) + return response + + + + +### ------ Begin Scan Views ------ ### + + + + class Scans(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post', 'get',] pagination_class = LimitOffsetPagination @@ -92,25 +172,15 @@ def get(self, request): return response + + class ScanDetail(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete',] def get(self, request, id): - scan = get_object_or_404(Scan, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if scan.site.account != account: - data = {'reason': 'you cannot retrieve Scans of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = ScanSerializer(scan, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_scan(request, id) + return response def delete(self, request, id): @@ -118,8 +188,10 @@ def delete(self, request, id): return response + + class ScanLean(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', ] def get(self, request, id): @@ -127,17 +199,21 @@ def get(self, request, id): return response -class ScanDelay(APIView): - permission_classes = (AllowAny,) + + +class ScansCreate(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): - response = create_scan(request, delay=True) + response = create_many_scans(request) return response + + class ScansDelete(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): @@ -147,9 +223,24 @@ def post(self, request): +class ScansZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_scans_zapier(request) + return response + + + + +### ------ Begin Test Views ------ ### + + + class Tests(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post', 'get',] pagination_class = LimitOffsetPagination @@ -162,33 +253,25 @@ def get(self, request): return response + + class TestDetail(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete',] def get(self, request, id): - test = get_object_or_404(Test, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if test.site.account != account: - data = {'reason': 'you cannot retrieve Tests of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = TestSerializer(test, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_test(request, id) + return response def delete(self, request, id): response = delete_test(request, id) return response + + class TestLean(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get',] def get(self, request, id): @@ -196,17 +279,21 @@ def get(self, request, id): return response -class TestDelay(APIView): - permission_classes = (AllowAny,) + + +class TestsCreate(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): - response = create_test(request, delay=True) + response = create_many_tests(request) return response + + class TestsDelete(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): @@ -216,10 +303,24 @@ def post(self, request): +class TestsZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_tests_zapier(request) + return response + + + + +### ------ Begin Schedule Views ------ ### + + class Schedules(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post', 'get'] def post(self, request): @@ -231,25 +332,15 @@ def get(self, request): return response + + class ScheduleDetail(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete'] def get(self, request, id): - schedule = get_object_or_404(Schedule, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if schedule.site.account != account: - data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = ScheduleSerializer(schedule, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_schedule(request, id) + return response def delete(self, request, id): response = delete_schedule(request, id) @@ -258,51 +349,82 @@ def delete(self, request, id): +class ScheduleRun(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post'] -class Automations(APIView): - permission_classes = (AllowAny,) + def post(self, request): + response = run_schedule(request) + return response + + + + +class SchedulesUpdate(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = update_many_schedules(request) + return response + + + + +class SchedulesDelete(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_schedules(request) + return response + + + + +### ------ Begin Alert Views ------ ### + + + + +class Alerts(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'post'] pagination_class = LimitOffsetPagination def post(self, request): - response = create_or_update_automation(request) + response = create_or_update_alert(request) return response def get(self, request): - response = get_automations(request) + response = get_alerts(request) return response -class AutomationDetail(APIView): - permission_classes = (AllowAny,) + + +class AlertDetail(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete'] def get(self, request, id): - automation = get_object_or_404(Automation, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if automation.user != request.user: - data = {'reason': 'you cannot retrieve Automations you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = AutomationSerializer(automation, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_alert(request, id) + return response def delete(self, request, id): - response = delete_automation(request, id) + response = delete_alert(request, id) return response +### ------ Begin Report Views ------ ### + + + class Reports(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post', 'get'] def post(self, request): @@ -315,25 +437,14 @@ def get(self, request): + class ReportDetail(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete'] def get(self, request, id): - report = get_object_or_404(Report, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if report.account != account: - data = {'reason': 'you cannot retrieve Reports you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = ReportSerializer(report, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_report(request, id) + return response def delete(self, request, id): response = delete_report(request, id) @@ -342,12 +453,23 @@ def delete(self, request, id): +class ExportReport(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = export_report(request) + return response + + + +### ------ Begin Case Views ------ ### class Cases(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['post', 'get'] def post(self, request): @@ -360,8 +482,9 @@ def get(self, request): + class CasesSearch(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get'] def get(self, request): @@ -370,25 +493,24 @@ def get(self, request): + +class CasePreRun(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post'] + + def post(self, request): + response = case_pre_run(request) + return response + + + class CaseDetail(APIView): - permission_classes = (AllowAny,) + permission_classes = (IsAuthenticated,) http_method_names = ['get', 'delete'] def get(self, request, id): - case = get_object_or_404(Case, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if case.account != account: - data = {'reason': 'you cannot retrieve Cases you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = CaseSerializer(case, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) + response = get_case(request, id) + return response def delete(self, request, id): response = delete_case(request, id) @@ -396,147 +518,564 @@ def delete(self, request, id): -class Testcases(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post', 'get'] + +class AutoCases(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post'] def post(self, request): - response = create_testcase(request) - return response - - def get(self, request): - response = get_testcases(request) + response = create_auto_cases(request) return response -class TestcaseDelay(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] + +class CopyCases(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post'] def post(self, request): - response = create_testcase(request, delay=True) + response = copy_case(request) return response -class TestcaseDetail(APIView): - permission_classes = (AllowAny,) - http_method_names = ['get', 'delete'] - def get(self, request, id): - testcase = get_object_or_404(Testcase, pk=id) - user = request.user - account = Member.objects.get(user=user).account - - if testcase.account != account: - data = {'reason': 'you cannot retrieve Testcases you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = TestcaseSerializer(testcase, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) +class CasesDelete(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] - def delete(self, request, id): - response = delete_testcase(request, id) + def post(self, request): + response = delete_many_cases(request) return response +class CasesZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] -class Logs(APIView): - permission_classes = (AllowAny,) - http_method_names = ['get',] - pagination_class = LimitOffsetPagination - def get(self, request): - response = get_logs(request) - return response - - -class LogDetail(APIView): - permission_classes = (AllowAny,) - http_method_names = ['get',] + response = get_cases_zapier(request) + return response - def get(self, request, id): - log = get_object_or_404(Log, pk=id) - if log.user != request.user: - data = {'reason': 'you cannot retrieve Logs you do not own',} - record_api_call(request, data, '403') - return Response(data, status=status.HTTP_403_FORBIDDEN) - - serializer_context = {'request': request,} - serialized = LogSerializer(log, context=serializer_context) - data = serialized.data - return Response(data, status=status.HTTP_200_OK) -class HomeStats(APIView): - permission_classes = (AllowAny,) - http_method_names = ['get',] +### ------ Begin CaseRun Views ------ ### - def get(self, request): - response = get_home_stats(request) - return response +class CaseRuns(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] -class Processes(APIView): - permission_classes = (AllowAny,) - http_method_names = ['get'] + def post(self, request): + response = create_caserun(request) + return response def get(self, request): - response = get_processes(request) + response = get_caseruns(request) return response -class ProcessDetail(APIView): - permission_classes = (AllowAny,) - http_method_names = ['get',] - def get(self, request, id): - if not Process.objects.filter(id=id).exists(): - data = {'reason': 'process with that id does not exist',} - record_api_call(request, data, '404') - return Response(data, status=status.HTTP_404_NOT_FOUND) - - proc = Process.objects.get(id=id) - serializer_context = {'request': request,} - serialized = ProcessSerializer(proc, context=serializer_context) - data = serialized.data - record_api_call(request, data, '200') - return Response(data, status=status.HTTP_200_OK) +class CaseRunDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + def get(self, request, id): + response = get_caserun(request, id) + return response -class WordPressMigrateSite(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - - def post(self, request): - response = migrate_site(request, delay=False) + def delete(self, request, id): + response = delete_caserun(request, id) return response -class WordPressMigrateSiteDelay(APIView): - permission_classes = (AllowAny,) - http_method_names = ['post',] - - def post(self, request): - response = migrate_site(request, delay=True) - return response -class SiteScreenshot(APIView): +class CaseRunsZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_caseruns_zapier(request) + return response + + + + +### ------ Begin Flow Views ------ ### + + + + +class Flows(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_flow(request) + return response + + def get(self, request): + response = get_flows(request) + return response + + + + +class FlowsSearch(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = search_flows(request) + return response + + + + +class FlowDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_flow(request, id) + return response + + def delete(self, request, id): + response = delete_flow(request, id) + return response + + + + +class CopyFlows(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post'] + + def post(self, request): + response = copy_flow(request) + return response + + + + +class FlowsDelete(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_flows(request) + return response + + + + +class FlowsZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_flows_zapier(request) + return response + + + + +### ------ Begin FlowRun Views ------ ### + + + + +class FlowRuns(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_flowrun(request) + return response + + def get(self, request): + response = get_flowruns(request) + return response + + + + +class FlowRunDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_flowrun(request, id) + return response + + def delete(self, request, id): + response = delete_flowrun(request, id) + return response + + + + +class FlowRunsZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_flowruns_zapier(request) + return response + + + + +### ------ Begin Issue Views ------ ### + + + + +class Issues(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_issue(request) + return response + + def get(self, request): + response = get_issues(request) + return response + + + + +class IssueGenerate(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = generate_issue(request) + return response + + + + +class IssuesSearch(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = search_issues(request) + return response + + + + +class IssueDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_issue(request, id) + return response + + def delete(self, request, id): + response = delete_issue(request, id) + return response + + + + +class IssuesUpdate(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = update_many_issues(request) + return response + + + + +class IssuesDelete(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = delete_many_issues(request) + return response + + + + +class IssuesZapier(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_issues_zapier(request) + return response + + + + +### ------ Begin Secret Views ------ ### + + + + +class Secrets(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_secret(request) + return response + + def get(self, request): + response = get_secrets(request) + return response + + + + +class SecretDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_secret(request, id) + return response + + def delete(self, request, id): + response = delete_secret(request, id) + return response + + + + +class SecretsAll(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_secrets_all(request) + return response + + + + +### ------ Begin Chat Views ------ ### + + + + +class Chats(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post', 'get'] + + def post(self, request): + response = create_or_update_chat(request) + return response + + def get(self, request): + response = get_chats(request) + return response + + + + +class ChatDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_chat(request, id) + return response + + def delete(self, request, id): + response = delete_chat(request, id) + return response + + + + +### ------ Begin Log Views ------ ### + + + + +class Logs(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + pagination_class = LimitOffsetPagination + + def get(self, request): + response = get_logs(request) + return response + + + + +class LogDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request, id): + response = get_log(request, id) + return response + + + + +### ------ Begin Process Views ------ ### + + + + +class Processes(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get'] + + def get(self, request): + response = get_processes(request) + return response + + + + +class ProcessDetail(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get', 'delete'] + + def get(self, request, id): + response = get_process(request, id) + return response + + def delete(self, request, id): + response = delete_process(request, id) + return response + + + + +### ------ Begin Search Views ------ ### + + + + +class Search(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = search_resources(request) + return response + + + + +class Tag(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_tags(request) + return response + + + + +class Device(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_devices(request) + return response + + + + +### ------ Begin Metrics Views ------ ### + + + + +class HomeMetrics(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_home_metrics(request) + return response + + + + +class SiteMetrics(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_site_metrics(request) + return response + + + + +class PageMetrics(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['get',] + + def get(self, request): + response = get_page_metrics(request) + return response + + + + +class CeleryMetrics(APIView): + authentication_classes = [] permission_classes = (AllowAny,) + http_method_names = ['get',] + + def get(self, request): + response = get_celery_metrics(request) + return response + + + + +### ------ Begin Beta Views ------ ### + + + + +class WordPressMigrateSite(APIView): + permission_classes = (IsAuthenticated,) + http_method_names = ['post',] + + def post(self, request): + response = migrate_site(request) + return response + + + + +class SiteScreenshot(APIView): + permission_classes = (IsAuthenticated,) http_method_names = ['post',] def post(self, request): response = create_site_screenshot(request) - return response \ No newline at end of file + return response + + + + diff --git a/app/scanerr/__init__.py b/app/cursion/__init__.py similarity index 100% rename from app/scanerr/__init__.py rename to app/cursion/__init__.py diff --git a/app/scanerr/asgi.py b/app/cursion/asgi.py similarity index 75% rename from app/scanerr/asgi.py rename to app/cursion/asgi.py index fa484bf4..80cdefbc 100644 --- a/app/scanerr/asgi.py +++ b/app/cursion/asgi.py @@ -1,5 +1,5 @@ """ -ASGI config for scanerr project. +ASGI config for cursion project. It exposes the ASGI callable as a module-level variable named ``application``. @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings') application = get_asgi_application() diff --git a/app/cursion/celery.py b/app/cursion/celery.py new file mode 100644 index 00000000..6ab129bd --- /dev/null +++ b/app/cursion/celery.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import, unicode_literals +from celery import Celery +from celery.signals import worker_shutdown +from django.conf import settings +import cursion, os, time + + + + + + +# setting DJANGO_SETTINGS_MODULE to cursion.settings +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings') + +# init celery +app = Celery('cursion') + +# configure namespace +app.config_from_object('django.conf:settings', namespace='CELERY') + +# celery and beat configs +app.conf.update( + task_acks_late=True, + worker_prefetch_multiplier=1, + worker_hijack_root_logger=False, + task_always_eager=False, + task_reject_on_worker_lost=True, + worker_cancel_long_running_tasks_on_connection_loss=True, + worker_max_tasks_per_child=100 +) + +# setting tasks to auto-discover +app.autodiscover_tasks() + +# setting debug +@app.task(bind=False) +def debug_task(self): + print(f'Request: {self.request}') + +# notify of SIGTERM +@worker_shutdown.connect +def on_worker_shutdown(**kwargs): + print(f'- WORKER SHUTTING DOWN - \n{kwargs}') + + + + diff --git a/app/scanerr/settings.py b/app/cursion/settings.py similarity index 62% rename from app/scanerr/settings.py rename to app/cursion/settings.py index 1d4b50c8..dc9f9588 100644 --- a/app/scanerr/settings.py +++ b/app/cursion/settings.py @@ -1,39 +1,59 @@ """ -Django settings for Scanerr project. +Django settings for Cursion project. -Generated by 'django-admin startproject' using Django 3.2.3. +Generated by 'django-admin startproject' using Django 5.0.6. For more information on this file, see -https://docs.djangoproject.com/en/3.2/topics/settings/ +https://docs.djangoproject.com/en/5.0/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/3.2/ref/settings/ +https://docs.djangoproject.com/en/5.0/ref/settings/ """ from pathlib import Path from datetime import timedelta import os +from kombu import Exchange, Queue + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent + # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = os.environ.get('SECRET_KEY') + # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = True if os.environ.get('DEBUG') == 'True' else False + -ALLOWED_HOSTS = ['*'] +# Specifies app and billing behavior +MODE = os.environ.get('MODE') + + +# Network settings +CORS_ORIGIN_ALLOW_ALL = True +DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +ALLOWED_HOSTS = [os.environ.get('DJANGO_ALLOWED_HOSTS')] + + +# URLs & location CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT') LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT') API_URL_ROOT = os.environ.get('API_URL_ROOT') -CORS_ORIGIN_ALLOW_ALL = True -DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880 +MCP_URL_ROOT = os.environ.get('MCP_URL_ROOT') +YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT') +LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT') +LOCATION = os.environ.get('LOCATION') -SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") -# Application definition +# Cursion.landing API KEY +LANDING_API_KEY = os.environ.get('LANDING_API_KEY') + +# Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', @@ -49,7 +69,6 @@ 'markdownify.apps.MarkdownifyConfig', 'storages', ] - MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', @@ -61,9 +80,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'corsheaders.middleware.CorsMiddleware', ] - -ROOT_URLCONF = 'scanerr.urls' - +ROOT_URLCONF = 'cursion.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', @@ -79,8 +96,7 @@ }, }, ] - -WSGI_APPLICATION = 'scanerr.wsgi.application' +WSGI_APPLICATION = 'cursion.wsgi.application' # Database @@ -97,7 +113,6 @@ } - # Password validation # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ @@ -128,7 +143,6 @@ 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 'PAGE_SIZE': 10, } - SIMPLE_JWT = { 'ACCESS_TOKEN_LIFETIME': timedelta(hours=24), 'REFRESH_TOKEN_LIFETIME': timedelta(hours=36), @@ -137,42 +151,28 @@ # Internationalization # https://docs.djangoproject.com/en/3.2/topics/i18n/ - LANGUAGE_CODE = 'en-us' - TIME_ZONE = 'UTC' - USE_I18N = True - USE_L10N = True - USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ -STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, "static") +STATIC_URL = '/staticfiles/' +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') -# needed for deployments without nginx -STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" -### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ### -# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' -# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN') -# STATIC_ROOT = 'static' -# MEDIA_ROOT = 'media' -# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/" -# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/" -# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH') -# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN') +# Static file service without nginx +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3' AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') + # Configure which endpoint to send files to, and retrieve files from. AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME') AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME') @@ -189,11 +189,35 @@ } +# Redis and Celery Config +CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379') + +# Celery queues +CELERY_QUEUE_SCHEDULED = os.environ.get('CELERY_QUEUE_SCHEDULED', 'scheduled') +CELERY_QUEUE_ON_DEMAND = os.environ.get('CELERY_QUEUE_ON_DEMAND', 'on_demand') + +# Default to scheduled so interactive work can reserve capacity via the on_demand workers +CELERY_TASK_DEFAULT_QUEUE = os.environ.get('CELERY_TASK_DEFAULT_QUEUE', CELERY_QUEUE_SCHEDULED) +CELERY_TASK_DEFAULT_EXCHANGE = os.environ.get('CELERY_TASK_DEFAULT_EXCHANGE', CELERY_TASK_DEFAULT_QUEUE) +CELERY_TASK_DEFAULT_ROUTING_KEY = os.environ.get('CELERY_TASK_DEFAULT_ROUTING_KEY', CELERY_TASK_DEFAULT_QUEUE) +CELERY_TASK_CREATE_MISSING_QUEUES = True +CELERY_TASK_QUEUES = ( + Queue(CELERY_QUEUE_SCHEDULED, Exchange(CELERY_QUEUE_SCHEDULED), routing_key=CELERY_QUEUE_SCHEDULED), + Queue(CELERY_QUEUE_ON_DEMAND, Exchange(CELERY_QUEUE_ON_DEMAND), routing_key=CELERY_QUEUE_ON_DEMAND), +) + + +# RabbitMQ and Celery Config +# CELERY_BROKER_URL = 'amqp://rabbitmq' -# Redis and Celery Conf -CELERY_BROKER_URL = "redis://redis:6379" -CELERY_RESULT_BACKEND = "redis://redis:6379" +# Django Caching framework with Redis +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.redis.RedisCache', + 'LOCATION': 'redis://redis:6379', + } +} # Default primary key field type @@ -201,7 +225,7 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -# email +# Email EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' EMAIL_HOST = os.environ.get('EMAIL_HOST') EMAIL_PORT = os.environ.get('EMAIL_PORT') @@ -211,22 +235,73 @@ # Sendgrid configs +SENDGRID_EMAIL = os.environ.get('SENDGRID_EMAIL') SENDGRID_API_KEY = os.environ.get('SENDGRID_API_KEY') DEFAULT_TEMPLATE = os.environ.get('DEFAULT_TEMPLATE') DEFAULT_TEMPLATE_NO_BUTTON = os.environ.get('DEFAULT_TEMPLATE_NO_BUTTON') AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE') +# Twilio configs +TWILIO_SID = os.environ.get('TWILIO_SID') +TWILIO_AUTH_TOKEN = os.environ.get('TWILIO_AUTH_TOKEN') +TWILIO_NUMBER = os.environ.get('TWILIO_NUMBER') -# google oAuth2 + +# Google oAuth2 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID') GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET') -# stripe keys +# Google API key +GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY') + + +# OpenAI's GPT API key +GPT_API_KEY = os.environ.get('GPT_API_KEY') + + +# Encryption Key +SECRETS_KEY = os.environ.get('SECRETS_KEY') + + +# Stripe keys if os.environ.get('STRIPE_ENV') == 'prod': STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE') STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE') if os.environ.get('STRIPE_ENV') == 'dev': STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST') - STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST') \ No newline at end of file + STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST') + + +# Global configs +CONFIGS = { + 'window_size': '1920,1080', + 'browser': 'chrome', + 'device': 'Windows 10 PC', + 'location': 'us', + 'mask_ids': None, + 'interval': 1, + 'min_wait_time': 3, + 'max_wait_time': 30, + 'timeout': 300, + 'disable_animations': True, + 'auto_height': True, + 'create_issue': True, + 'end_on_fail': True, + 'ai_analysis': False, + 'api_priority': False, +} + + +# Global Test.threshold +TEST_THRESHOLD = 95 + + +# Global Scan & Test types +TYPES = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'] + + +# Global max attempts +MAX_ATTEMPTS = 3 + diff --git a/app/scanerr/urls.py b/app/cursion/urls.py similarity index 98% rename from app/scanerr/urls.py rename to app/cursion/urls.py index 50d33dcf..4574aef0 100644 --- a/app/scanerr/urls.py +++ b/app/cursion/urls.py @@ -3,6 +3,9 @@ + + + urlpatterns = [ path('admin/', admin.site.urls), path('', include('api.urls')), diff --git a/app/scanerr/wsgi.py b/app/cursion/wsgi.py similarity index 75% rename from app/scanerr/wsgi.py rename to app/cursion/wsgi.py index bc13ad63..7587bfe8 100644 --- a/app/scanerr/wsgi.py +++ b/app/cursion/wsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for scanerr project. +WSGI config for cursion project. It exposes the WSGI callable as a module-level variable named ``application``. @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings') application = get_wsgi_application() diff --git a/app/manage.py b/app/manage.py index d4057aa5..00ce80c6 100755 --- a/app/manage.py +++ b/app/manage.py @@ -6,7 +6,7 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings') try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py deleted file mode 100644 index 08e5ae7e..00000000 --- a/app/scanerr/celery.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import absolute_import, unicode_literals -from celery import Celery -from django.conf import settings -import scanerr, os - - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings') - -app = Celery('scanerr') -app.config_from_object('django.conf:settings', namespace='CELERY') -app.autodiscover_tasks() - - -@app.task(bind=False) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) - diff --git a/archive/Dockerfile b/archive/Dockerfile new file mode 100644 index 00000000..cf1102d9 --- /dev/null +++ b/archive/Dockerfile @@ -0,0 +1,81 @@ +# pull main python image +FROM python:3.9-slim +ENV PYTHONUNBUFFERED 1 + +# increasing allocated memory to node +ENV NODE_OPTIONS=--max_old_space_size=20000 +ENV NODE_OPTIONS="--max-old-space-size=20000" +ENV GENERATE_SOURCEMAP=false + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + +# telling phantomas where Chrome binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium +ENV DOCKERIZED=yes + +# Set up the Chromium environment +ENV XDG_CONFIG_HOME=/tmp/.chromium +ENV XDG_CACHE_HOME=/tmp/.chromium + +# create the app user +RUN addgroup --system app && adduser --system app + +# installing python3 & pip +RUN apt-get update && apt-get install -y python3 python3-pip + +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ + libfontconfig firefox-esr apt-transport-https software-properties-common + +# installing google-chrome-stable +RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +RUN apt-get install -y ./google-chrome-stable_current_amd64.deb +RUN rm google-chrome-stable_current_amd64.deb + +# Download and install Microsoft Edge +RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \ + install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \ + sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \ + /etc/apt/sources.list.d/microsoft-edge.list' && \ + apt-get update && apt-get install -y microsoft-edge-stable && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg + +# installing node and npm --> n lts +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n \ + && n lts + +# cleaning npm +RUN npm cache clean --force + +# installing lighthouse +RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools + +# setting --no-sandbox & --disable-dev-shm-usage +RUN chromium --no-sandbox --version +RUN chromium --disable-dev-shm-usage --version + +# installing requirements +COPY ./setup/requirements/requirements.txt /requirements.txt +RUN python3 -m pip install -r /requirements.txt + +# removing chromium config +RUN rm -rf ~/.config/chromium + +# setting working dir +RUN mkdir /app +COPY ./app /app +WORKDIR /app + +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/chromium +RUN chown -R app:app /usr/bin/chromedriver +RUN chmod +x /usr/bin/chromedriver + +# staring up services +COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh" +ENTRYPOINT [ "/remote-entrypoint.sh" ] + diff --git a/archive/Dockerfile.local b/archive/Dockerfile.local new file mode 100644 index 00000000..5ac483bf --- /dev/null +++ b/archive/Dockerfile.local @@ -0,0 +1,74 @@ +# pull main ubuntu image and set platform to linux/amd64 +FROM --platform=linux/amd64 ubuntu:latest +ENV DOCKER_DEFAULT_PLATFORM linux/amd64 +ENV PYTHONUNBUFFERED 1 +ENV DEBIAN_FRONTEND noninteractive + +# increasing allocated memory to node +ENV NODE_OPTIONS=--max_old_space_size=7000 +ENV NODE_OPTIONS="--max-old-space-size=7000" + +# telling Puppeteer to skip installing Chrome +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true + +# telling phantomas where Chrome binary is and that we're in docker +ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable +ENV DOCKERIZED=yes + +# create the app user +RUN groupadd --system app & useradd --system app + +# installing python3 & pip +RUN apt-get update && apt-get install -y python3.10 python3-pip + +# installing system deps +RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \ + gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox \ + apt-transport-https software-properties-common libglib2.0-0 libsm6 \ + libxrender1 libxext6 libgl1 + +# installing google-chrome-stable +RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +RUN apt-get install -y ./google-chrome-stable_current_amd64.deb +RUN rm google-chrome-stable_current_amd64.deb + +# install microsoft-edge-stable +RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \ + install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \ + sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \ + /etc/apt/sources.list.d/microsoft-edge.list' && \ + apt-get update && apt-get install -y microsoft-edge-stable && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg + +# installing node and npm +RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \ + && npm install -g n && n lts + +# begin npm portion +RUN npm cache clean --force + +# installing lighthouse & yellowlabtools +RUN npm install -g lighthouse lighthouse-plugin-crux lodash + +# setting --no-sandbox for Phantomas +RUN google-chrome-stable --no-sandbox --version + +# installing requirements +RUN python3 -m pip install --break-system-packages --upgrade setuptools +COPY ./setup/requirements/requirements-staging.txt /requirements-staging.txt +RUN python3 -m pip install --break-system-packages -r /requirements-staging.txt + +# setting working dir +RUN mkdir /app +COPY ./app /app +WORKDIR /app + +# setting ownership +RUN chown -R app:app /app +RUN chown -R app:app /usr/bin/google-chrome-stable +RUN chown -R app:app /usr/bin/firefox +RUN chown -R app:app /usr/bin/microsoft-edge-stable + +# staring up services +COPY ./setup/scripts/local-entrypoint.sh "/local-entrypoint.sh" +ENTRYPOINT [ "/local-entrypoint.sh" ] diff --git a/archive/docker-compose.dev.yml b/archive/docker-compose.dev.yml new file mode 100644 index 00000000..51c7796d --- /dev/null +++ b/archive/docker-compose.dev.yml @@ -0,0 +1,140 @@ +services: + + + app: + container_name: scanerr-app + hostname: scanerr-app + restart: always + privileged: true + init: true + build: + context: . + dockerfile: Dockerfile + entrypoint: ["/remote-entrypoint.sh", "app"] + expose: + - 8000 + env_file: + - ./env/.env.dev + volumes: + - ./app:/app + - static_volume:/app/static + depends_on: + - db + + + db: + container_name: scanerr-db + hostname: scanerr-db + image: postgres:14-alpine + ports: + - "5432" + env_file: + - ./env/.env.dev + volumes: + - pgdata:/var/lib/postgresql/data + + + redis: + container_name: scanerr-redis + hostname: scanerr-redis + image: redis:alpine + ports: + - "6379" + + + celery: + container_name: scanerr-celery + hostname: scanerr-celery + privileged: true + restart: always + build: + context: . + dockerfile: Dockerfile + entrypoint: ["/remote-entrypoint.sh", "celery"] + env_file: + - ./env/.env.dev + volumes: + - ./app:/scanerr + depends_on: + - redis + - app + - db + + + beat: + container_name: scanerr-beat + hostname: scanerr-beat + privileged: true + restart: always + build: + context: . + dockerfile: Dockerfile + entrypoint: ["/remote-entrypoint.sh", "beat"] + volumes: + - ./app:/scanerr + env_file: + - ./env/.env.dev + depends_on: + - redis + - celery + - app + - db + + + yellowlab: + container_name: yellowlab + hostname: yellowlab + privileged: true + restart: always + image: scanerr/ylt + ports: + - 8383:8383 + depends_on: + - redis + - celery + - app + - db + + + nginx-proxy: + container_name: nginx-proxy + hostname: nginx-proxy + build: nginx + restart: always + ports: + - 443:443 + - 80:80 + volumes: + - static_volume:/app/static + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + depends_on: + - app + + + nginx-proxy-letsencrypt: + container_name: nginx-proxy-letsencrypt + hostname: nginx-proxy-letsencrypt + image: nginxproxy/acme-companion + environment: + - DEFAULT_EMAIL=youremail@yourdomain.com + - NGINX_PROXY_CONTAINER=nginx-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - letsencrypt-acme:/etc/acme.sh + depends_on: + - nginx-proxy + + +volumes: + static_volume: + letsencrypt-acme: + pgdata: + certs: + html: + vhost: \ No newline at end of file diff --git a/archive/docker-compose.prod.yml b/archive/docker-compose.prod.yml new file mode 100644 index 00000000..19579682 --- /dev/null +++ b/archive/docker-compose.prod.yml @@ -0,0 +1,146 @@ +services: + + + client: + image: cursiondev/client + env_file: + - .env.prod + ports: + - "3000:3000" + + + app: + container_name: cursion-app + hostname: cursion-app + restart: always + platform: linux/amd64 + privileged: true + init: true + image: cursiondev/server + entrypoint: ["/entrypoint.sh", "app", "remote"] + expose: + - 8000 + env_file: + - .env.prod + volumes: + - app:/app + - static_volume:/app/static + depends_on: + - db + + + db: + container_name: cursion-db + hostname: cursion-db + image: postgres:14-alpine + ports: + - "5432" + env_file: + - .env.prod + volumes: + - pgdata:/var/lib/postgresql/data + + + redis: + container_name: cursion-redis + hostname: cursion-redis + image: redis:alpine + ports: + - "6379" + + + celery: + container_name: cursion-celery + hostname: cursion-celery + privileged: true + restart: always + image: cursiondev/server + entrypoint: ["/entrypoint.sh", "celery"] + env_file: + - .env.prod + volumes: + - celery:/app + depends_on: + - redis + - app + - db + + + beat: + container_name: cursion-beat + hostname: cursion-beat + privileged: true + restart: always + image: cursiondev/server + entrypoint: ["/entrypoint.sh", "beat"] + volumes: + - beat:/app + env_file: + - .env.prod + depends_on: + - redis + - celery + - app + - db + + + yellowlab: + container_name: yellowlab + hostname: yellowlab + privileged: true + restart: always + image: cursiondev/ylt + ports: + - 8383:8383 + depends_on: + - redis + - celery + - app + - db + + + nginx-proxy: + container_name: nginx-proxy + hostname: nginx-proxy + image: cursiondev/nginx + restart: always + ports: + - 443:443 + - 80:80 + volumes: + - static_volume:/app/static + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + depends_on: + - app + + + nginx-proxy-letsencrypt: + container_name: nginx-proxy-letsencrypt + hostname: nginx-proxy-letsencrypt + image: nginxproxy/acme-companion + environment: + - DEFAULT_EMAIL=youremail@yourdomain.com + - NGINX_PROXY_CONTAINER=nginx-proxy + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - letsencrypt-acme:/etc/acme.sh + depends_on: + - nginx-proxy + + +volumes: + app: + celery: + beat: + static_volume: + letsencrypt-acme: + pgdata: + certs: + html: + vhost: \ No newline at end of file diff --git a/archive/remote-entrypoint.sh b/archive/remote-entrypoint.sh new file mode 100755 index 00000000..c11229a2 --- /dev/null +++ b/archive/remote-entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# spin up app in remote env +if [[ $1 == *"app"* ]] +then + python3 manage.py wait_for_db && + python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input && + python3 manage.py collectstatic --no-input && + python3 manage.py create_admin && + python3 manage.py driver_test && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000 +fi + +# spin up celery in remote env +if [[ $1 == *"celery"* ]] +then + python3 manage.py wait_for_db && + echo "pausing for migrations to complete..." && sleep 7s && + celery -A scanerr worker -E --loglevel=info -O fair +fi + +# spin up celery beat in remote env +if [[ $1 == *"beat"* ]] +then + python3 manage.py wait_for_db && + echo "pausing for migrations to complete..." && sleep 7s && + celery -A scanerr beat --scheduler django --loglevel=info +fi diff --git a/requirements.txt b/archive/requirements.txt similarity index 85% rename from requirements.txt rename to archive/requirements.txt index df9fe6b2..46d35ecf 100644 --- a/requirements.txt +++ b/archive/requirements.txt @@ -1,9 +1,10 @@ amqp==5.0.6 asgiref==3.3.4 +beautifulsoup4==4.12.2 billiard==3.6.4.0 boto3==1.20.32 celery==5.1.0 -certifi==2021.5.30 +certifi==2023.7.22 chardet==4.0.0 click==7.1.2 click-didyoumean==0.0.3 @@ -21,28 +22,29 @@ docker==5.0.0 gunicorn==20.1.0 humanize==3.7.0 idna==2.10 +imutils==0.5.4 kombu==5.1.0 Markdown==3.3.4 numpy==1.22.3 +openai==1.35.14 opencv-python==4.5.5.64 -Pillow==9.0.0 +Pillow==9.0.1 prometheus-client==0.8.0 prompt-toolkit==3.0.18 psycopg2==2.8.6 pyjwt==2.1.0 -pyppeteer==1.0.2 pytz==2021.1 redis==3.5.3 requests==2.25.1 reportlab==3.6.6 +scikit-image==0.21.0 scipy==1.8.0 -selenium==4.1.3 +selenium==4.18.1 sendgrid==6.9.7 -sewar==0.4.4 six==1.16.0 slack-sdk==3.11.2 sqlparse==0.4.1 -stripe==2.60.0 +stripe==8.0.0 tornado==6.1 twilio==7.3.0 urllib3==1.26.5 diff --git a/commands b/commands deleted file mode 100644 index a6764d18..00000000 --- a/commands +++ /dev/null @@ -1,22 +0,0 @@ -### spins up container on localhost ### -docker compose up --build - -### spins down container on localhost ### -docker compose down - - - -### spins up the container for production ### -docker compose -f docker-compose.prod.yml up -d --build - -### spins down the container ### -docker compose -f docker-compose.prod.yml down - - - -### spins up the container for development ### -docker compose -f docker-compose.dev.yml up -d --build - -### spins down the container ### -docker compose -f docker-compose.dev.yml down - diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f09772ed..b86466c9 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,32 +1,46 @@ -version: '3' - +name: cursion services: - app: + + + client: + container_name: cursion-client + hostname: cursion-client + image: cursiondev/client:latest + platform: linux/amd64 + pull_policy: always + env_file: + - ./env/.env.client.dev + expose: + - "8080" + + + server: + container_name: cursion-server + hostname: cursion-server + platform: linux/amd64 privileged: true + restart: always init: true build: context: . - dockerfile: Dockerfile.prod - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations --no-input && - python3 manage.py migrate --no-input && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_s_test && - python3 manage.py driver_p_test && - gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + dockerfile: Dockerfile + entrypoint: ["/entrypoint.sh", "server", "remote"] expose: - - 8000 + - "8000" env_file: - ./env/.env.dev + volumes: + - server:/app + - static_volume:/app/staticfiles + depends_on: + - db - + db: - image: postgres:10-alpine + container_name: cursion-db + hostname: cursion-db + image: postgres:14-alpine + platform: linux/amd64 ports: - "5432" env_file: @@ -36,63 +50,136 @@ services: redis: + container_name: cursion-redis + hostname: cursion-redis image: redis:alpine + platform: linux/amd64 ports: - "6379" - celery: + celery-scheduled: + container_name: cursion-celery-scheduled + hostname: cursion-celery-scheduled + platform: linux/amd64 + privileged: true + restart: always + build: + context: . + dockerfile: Dockerfile + entrypoint: ["/entrypoint.sh", "celery", "scheduled"] + env_file: + - ./env/.env.dev + volumes: + - celery:/app + depends_on: + - redis + - server + - db + + celery-on-demand: + container_name: cursion-celery-on-demand + hostname: cursion-celery-on-demand + platform: linux/amd64 + privileged: true + restart: always + build: + context: . + dockerfile: Dockerfile + entrypoint: ["/entrypoint.sh", "celery", "on_demand"] + env_file: + - ./env/.env.dev + volumes: + - celery:/app + depends_on: + - redis + - server + - db + + + beat: + container_name: cursion-beat + hostname: cursion-beat + platform: linux/amd64 privileged: true restart: always build: context: . - dockerfile: Dockerfile.prod - command: celery -A scanerr worker --beat --scheduler django --loglevel=info + dockerfile: Dockerfile + entrypoint: ["/entrypoint.sh", "beat"] volumes: - - ./app:/scanerr + - beat:/app env_file: - ./env/.env.dev depends_on: - redis - - app + - celery-scheduled + - celery-on-demand + - server + - db + + + yellowlab: + container_name: yellowlab + hostname: yellowlab + privileged: true + restart: always + image: cursiondev/ylt + platform: linux/amd64 + ports: + - "8383:8383" + depends_on: + - redis + - celery-scheduled + - celery-on-demand + - server - db nginx-proxy: container_name: nginx-proxy - build: nginx + hostname: nginx-proxy + image: cursiondev/nginx + platform: linux/amd64 restart: always ports: - - 443:443 - - 80:80 + - "443:443" + - "80:80" volumes: - - static_volume:/app/static + - static_volume:/app/staticfiles - certs:/etc/nginx/certs - html:/usr/share/nginx/html - vhost:/etc/nginx/vhost.d - /var/run/docker.sock:/tmp/docker.sock:ro depends_on: - - app + - server + - client nginx-proxy-letsencrypt: - image: nginxproxy/acme-companion # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion + container_name: nginx-proxy-letsencrypt + hostname: nginx-proxy-letsencrypt + image: nginxproxy/acme-companion + platform: linux/amd64 env_file: - - ./env/.env.prod.proxy-companion + - ./env/.env.dev volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - certs:/etc/nginx/certs - html:/usr/share/nginx/html - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro - letsencrypt-acme:/etc/acme.sh depends_on: - nginx-proxy volumes: + server: static_volume: + celery: + beat: letsencrypt-acme: + pgdata: certs: html: vhost: - pgdata: \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml deleted file mode 100644 index cc375e35..00000000 --- a/docker-compose.prod.yml +++ /dev/null @@ -1,87 +0,0 @@ -version: '3' - -services: - app: - restart: always - privileged: true - init: true - build: - context: . - dockerfile: Dockerfile.prod - # image: landonr/scanerr-server - volumes: - - ./app:/app - - static_volume:/app/static - command: > - sh -c "python3 manage.py makemigrations --no-input && - python3 manage.py migrate --no-input && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_s_test && - python3 manage.py driver_p_test && - gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" - expose: - - 8000 - env_file: - - ./env/.env.prod - - redis: - image: redis:alpine - ports: - - "6379" - - celery: - privileged: true - restart: always - build: - context: . - dockerfile: Dockerfile.prod - # image: landonr/scanerr-server - command: celery -A scanerr worker --beat --scheduler django --loglevel=info - volumes: - - ./app:/scanerr - env_file: - - ./env/.env.prod - depends_on: - - redis - - app - - - nginx-proxy: - container_name: nginx-proxy - build: nginx - restart: always - ports: - - 443:443 - - 80:80 - volumes: - - static_volume:/app/static - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - - /var/run/docker.sock:/tmp/docker.sock:ro - depends_on: - - app - - - nginx-proxy-letsencrypt: - image: nginxproxy/acme-companion # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion - env_file: - - ./env/.env.prod.proxy-companion - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - certs:/etc/nginx/certs - - html:/usr/share/nginx/html - - vhost:/etc/nginx/vhost.d - - letsencrypt-acme:/etc/acme.sh - depends_on: - - nginx-proxy - - -volumes: - static_volume: - letsencrypt-acme: - certs: - html: - vhost: \ No newline at end of file diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml new file mode 100644 index 00000000..9c21a48c --- /dev/null +++ b/docker-compose.stage.yml @@ -0,0 +1,21 @@ +name: cursion +services: + + + server: + container_name: cursion-server + hostname: cursion-server + platform: linux/amd64 + privileged: true + init: true + restart: no + build: + context: . + dockerfile: Dockerfile + ports: + - "8000:8000" + entrypoint: ["/entrypoint.sh", "server", "stage"] + env_file: + - ./env/.env.stage + volumes: + - ./app:/app diff --git a/docker-compose.yml b/docker-compose.yml index a8f492c0..7931883b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,62 +1,120 @@ -version: '3' +name: cursion services: - app: + + server: + container_name: cursion-server + hostname: cursion-server + platform: linux/amd64 privileged: true + restart: no init: true - restart: always + image: cursion-app:local build: context: . - dockerfile: Dockerfile.prod - # image: landonr/scanerr-server + dockerfile: Dockerfile ports: - "8000:8000" - volumes: - - ./app:/app - command: > - sh -c "python3 manage.py makemigrations --no-input && - python3 manage.py migrate --no-input && - python3 manage.py collectstatic --no-input && - python3 manage.py wait_for_db && - python3 manage.py create_admin && - python3 manage.py driver_s_test && - python3 manage.py driver_p_test && - python3 manage.py runserver 0.0.0.0:8000" + entrypoint: ["/entrypoint.sh", "server", "local"] env_file: - ./env/.env.local + volumes: + - server:/app depends_on: - db + db: - image: postgres:10-alpine - ports: - - "5432" + container_name: cursion-db + hostname: cursion-db + image: postgres:14-alpine env_file: - - ./env/.env.dev + - ./env/.env.local volumes: - pgdata:/var/lib/postgresql/data + redis: + container_name: cursion-redis + hostname: cursion-redis image: redis:alpine ports: - "6379" - - celery: + + + celery-scheduled: + container_name: cursion-celery-scheduled + hostname: cursion-celery-scheduled + platform: linux/amd64 privileged: true - restart: always - build: - context: . - dockerfile: Dockerfile.prod - # image: landonr/scanerr-server - command: celery -A scanerr worker --beat --scheduler django --loglevel=info + restart: no + image: cursion-app:local + entrypoint: ["/entrypoint.sh", "celery", "scheduled"] volumes: - - ./app:/scanerr + - celery:/app env_file: - ./env/.env.local depends_on: + - redis + - server - db + + celery-on-demand: + container_name: cursion-celery-on-demand + hostname: cursion-celery-on-demand + platform: linux/amd64 + privileged: true + restart: no + image: cursion-app:local + entrypoint: ["/entrypoint.sh", "celery", "on_demand"] + volumes: + - celery:/app + env_file: + - ./env/.env.local + depends_on: - redis - - app + - server + - db + + + beat: + container_name: cursion-beat + hostname: cursion-beat + platform: linux/amd64 + privileged: true + restart: no + image: cursion-app:local + entrypoint: ["/entrypoint.sh", "beat"] + volumes: + - beat:/app + env_file: + - ./env/.env.local + depends_on: + - redis + - celery-scheduled + - celery-on-demand + - server + - db + + + yellowlab: + container_name: yellowlab + hostname: yellowlab + privileged: true + restart: no + image: cursiondev/ylt + ports: + - 8383:8383 + depends_on: + - redis + - celery-scheduled + - celery-on-demand + - server + - db + volumes: pgdata: + server: + celery: + beat: diff --git a/docker/docker-compose.remote.yml b/docker/docker-compose.remote.yml new file mode 100644 index 00000000..9e09e2a2 --- /dev/null +++ b/docker/docker-compose.remote.yml @@ -0,0 +1,131 @@ +services: + + + server: + container_name: cursion-server + hostname: cursion-server + image: cursiondev/server + platform: linux/amd64 + pull_policy: always + restart: always + privileged: true + init: true + entrypoint: ["/entrypoint.sh", "server", "remote"] + expose: + - "8000" + env_file: + - ./env/.env.remote + volumes: + - server:/app + - static_volume:/app/staticfiles + + + redis: + container_name: cursion-redis + hostname: cursion-redis + image: redis:alpine + platform: linux/amd64 + ports: + - "6379" + + + celery: + container_name: cursion-celery + hostname: cursion-celery + image: cursiondev/server + platform: linux/amd64 + pull_policy: always + privileged: true + restart: always + entrypoint: ["/entrypoint.sh", "celery"] + env_file: + - ./env/.env.remote + volumes: + - celery:/app + depends_on: + - redis + - server + + + beat: + container_name: cursion-beat + hostname: cursion-beat + image: cursiondev/server + platform: linux/amd64 + privileged: true + pull_policy: always + restart: always + entrypoint: ["/entrypoint.sh", "beat"] + volumes: + - beat:/app + env_file: + - ./env/.env.remote + depends_on: + - redis + - celery + - server + + + yellowlab: + container_name: yellowlab + hostname: yellowlab + image: cursiondev/ylt + pull_policy: always + platform: linux/amd64 + privileged: true + restart: always + ports: + - "8383:8383" + depends_on: + - redis + - celery + - server + + + nginx-proxy: + container_name: nginx-proxy + hostname: nginx-proxy + image: cursiondev/nginx + pull_policy: always + platform: linux/amd64 + restart: always + ports: + - "443:443" + - "80:80" + volumes: + - static_volume:/app/staticfiles + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + depends_on: + - server + + + nginx-proxy-letsencrypt: + container_name: nginx-proxy-letsencrypt + hostname: nginx-proxy-letsencrypt + image: nginxproxy/acme-companion + platform: linux/amd64 + privileged: true + env_file: + - ./env/.env.remote + volumes: + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhost:/etc/nginx/vhost.d + - /var/run/docker.sock:/var/run/docker.sock + - letsencrypt-acme:/etc/acme.sh + depends_on: + - nginx-proxy + + +volumes: + server: + static_volume: + celery: + beat: + letsencrypt-acme: + certs: + html: + vhost: \ No newline at end of file diff --git a/env/.env.dev.example b/env/.env.dev.example deleted file mode 100644 index f05120b0..00000000 --- a/env/.env.dev.example +++ /dev/null @@ -1,88 +0,0 @@ -# django -SECRET_KEY = ask-for-this-or-generate-yourself -CLIENT_URL_ROOT = https://app.example.io # example -API_URL_ROOT = https://api.example.io # example -LETSENCRYPT_HOST = api.example.io # example -VIRTUAL_HOST = api.example.io # example -VIRTUAL_PORT = 8000 -DJANGO_ALLOWED_HOSTS = * - - -# admin credentials -ADMIN_USER = fake # example -ADMIN_PASS = dontTryIt1234 # example -ADMIN_EMAIL = fake@example.com # example - - -# email credentials -EMAIL_HOST = smtp.gmail.com -EMAIL_PORT = 587 -EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com # example -EMAIL_HOST_PASSWORD = 1234456677888 # example - - -# database -DB_HOST=db -DB_NAME=app -DB_USER=postgres -DB_PASS=supersecretpassword -POSTGRES_DB=app -POSTGRES_USER=postgres -POSTGRES_PASSWORD=supersecretpassword - - -# paths -CHROMEDRIVER = /usr/bin/chromedriver -GOOGLECHROME = /usr/bin/google-chrome -CHROMIUM = /usr/bin/chromium - - -# stripe keys -STRIPE_PUBLIC_TEST = -STRIPE_PRIVATE_TEST = -STRIPE_PUBLIC_LIVE = -STRIPE_PRIVATE_LIVE = -STRIPE_ENV = dev - - -# google keys -GOOGLE_CRUX_KEY = - - -# OAuth keys -GOOGLE_OAUTH2_CLIENT_ID = -GOOGLE_OAUTH2_CLIENT_SECRET = - - -# twilio credentials -TWILIO_SID = -TWILIO_AUTH_TOKEN = -TWILIO_NUMBER = - - -# sendgrid configs -SENDGRID_API_KEY = -DEFAULT_TEMPLATE = -DEFAULT_TEMPLATE_NO_BUTTON = -AUTOMATION_TEMPLATE = - - -# slack credentials -SLACK_APP_ID = -SLACK_CLIENT_ID = -SLACK_CLIENT_SECRET = -SLACK_SIGNING_SECRET = -SLACK_VERIFICATION_TOKEN = -SLACK_BOT_TOKEN = - - -# s3 remote storage credentials -AWS_ACCESS_KEY_ID = -AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = storage-scanerr # example -AWS_S3_REGION_NAME = sfo3 # example -AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example -AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example -AWS_LOCATION = static -AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.local.example b/env/.env.local.example index 40b50b9c..f9668fe9 100644 --- a/env/.env.local.example +++ b/env/.env.local.example @@ -1,84 +1,107 @@ # high level django configs -SECRET_KEY = ask-for-this -CLIENT_URL_ROOT = http://localhost:3000 -CLIENT_URL_ROOT = http://localhost:3000 -API_URL_ROOT = http://localhost:8000 -DJANGO_ALLOWED_HOSTS = * +SECRET_KEY=abcdefghijklmno123456789 +DJANGO_ALLOWED_HOSTS=* +DEBUG=False +LOCATION=us +MODE=local +LANDING_URL_ROOT=https://cursion.dev +CLIENT_URL_ROOT=https://app.example.com # example +API_URL_ROOT=https://api.example.com # example +MCP_URL_ROOT=https://mcp.example.com # example +YELLOWLAB_ROOT=http://yellowlab:8383 # example +LIGHTHOUSE_ROOT=https://www.googleapis.com/pagespeedonline/v5/runPagespeed -# admin credentials -ADMIN_USER = fake # example -ADMIN_PASS = dontTryIt1234 # example -ADMIN_EMAIL = fake@example.com # example +# admin credentials +ADMIN_USER=admin +ADMIN_PASS=dontTryIt1234 # example +ADMIN_EMAIL=hello@example.com # example # email credentials -EMAIL_HOST = smtp.gmail.com -EMAIL_PORT = 587 -EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com # example -EMAIL_HOST_PASSWORD = 1234456677888 # example +EMAIL_HOST=smtp.gmail.com +EMAIL_PORT=587 +EMAIL_USE_TLS=True +EMAIL_HOST_USER=hello@example.com # example +EMAIL_HOST_PASSWORD=1234456677888 # example -# database configs +# database DB_HOST=db DB_NAME=app DB_USER=postgres -DB_PASS=supersecretpassword +DB_PASS=supersecretpassword # example POSTGRES_DB=app POSTGRES_USER=postgres -POSTGRES_PASSWORD=supersecretpassword +POSTGRES_PASSWORD=supersecretpassword # example # paths -CHROMEDRIVER = /usr/bin/chromedriver -GOOGLECHROME = /usr/bin/google-chrome -CHROMIUM = /usr/bin/chromium +CHROMEDRIVER=/usr/bin/chromedriver +CHROME_BROWSER=/usr/bin/chromium +FIREFOX_BROWSER=/usr/bin/firefox +EDGE_BROWSER=/usr/bin/microsoft-edge-stable # stripe keys -STRIPE_PUBLIC_TEST = -STRIPE_PRIVATE_TEST = +STRIPE_PUBLIC_TEST= +STRIPE_PRIVATE_TEST= +STRIPE_PUBLIC_LIVE= +STRIPE_PRIVATE_LIVE= +STRIPE_ENV=dev # google keys -GOOGLE_CRUX_KEY = +GOOGLE_CRUX_KEY= # OAuth keys -GOOGLE_OAUTH2_CLIENT_ID = -GOOGLE_OAUTH2_CLIENT_SECRET = +GOOGLE_OAUTH2_CLIENT_ID= +GOOGLE_OAUTH2_CLIENT_SECRET= # twilio credentials -TWILIO_SID = -TWILIO_AUTH_TOKEN = -TWILIO_NUMBER = +TWILIO_SID= +TWILIO_AUTH_TOKEN= +TWILIO_NUMBER=+13333333333 # sendgrid configs -SENDGRID_API_KEY = -DEFAULT_TEMPLATE = -DEFAULT_TEMPLATE_NO_BUTTON = -AUTOMATION_TEMPLATE = - +SENDGRID_EMAIL= +SENDGRID_API_KEY= +DEFAULT_TEMPLATE= +DEFAULT_TEMPLATE_NO_BUTTON= +AUTOMATION_TEMPLATE= # slack credentials -SLACK_APP_ID = -SLACK_CLIENT_ID = -SLACK_CLIENT_SECRET = -SLACK_SIGNING_SECRET = -SLACK_VERIFICATION_TOKEN = -SLACK_BOT_TOKEN = +SLACK_APP_ID= +SLACK_CLIENT_ID= +SLACK_CLIENT_SECRET= +SLACK_SIGNING_SECRET= +SLACK_VERIFICATION_TOKEN= +SLACK_BOT_TOKEN= # s3 remote storage credentials -AWS_ACCESS_KEY_ID = -AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = storage-scanerr # example -AWS_S3_REGION_NAME = sfo3 # example -AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example -AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example -AWS_LOCATION = static -AWS_DEFAULT_ACL = public-read \ No newline at end of file +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_STORAGE_BUCKET_NAME=storage-scanerr # example +AWS_S3_REGION_NAME=sfo3 # example +AWS_S3_ENDPOINT_URL=https://sfo3.digitaloceanspaces.com # example +AWS_S3_URL_PATH=https://storage-scanerr.sfo3.digitaloceanspaces.com # example +AWS_LOCATION=static +AWS_DEFAULT_ACL=public-read + + +# OpenAI API key +GPT_API_KEY=sk-123450989776124ni989wed23e9dub # example + + +# Self Hosted Cred +LICENSE_KEY=ask-for-this-cred-before-deploying + + +# Encryption Key (32 bytes) +SECRETS_KEY=generate-this-yourself-before-deployment + diff --git a/env/.env.prod.example b/env/.env.prod.example deleted file mode 100644 index 96d89f7c..00000000 --- a/env/.env.prod.example +++ /dev/null @@ -1,86 +0,0 @@ -# high level django configs -SECRET_KEY = ask-for-this-or-generate-yourself -CLIENT_URL_ROOT = https://app.example.io # example -LANDING_URL_ROOT = https://example.io # example -API_URL_ROOT = https://api.example.io # example -LETSENCRYPT_HOST = api.example.io # example -DJANGO_ALLOWED_HOSTS = * - - -# admin credentials -ADMIN_USER = fake # example -ADMIN_PASS = dontTryIt1234 # example -ADMIN_EMAIL = fake@example.com # example - - -# email credentials -EMAIL_HOST = smtp.gmail.com -EMAIL_PORT = 587 -EMAIL_USE_TLS = True -EMAIL_HOST_USER = fake@example.com # example -EMAIL_HOST_PASSWORD = 1234456677888 # example - - -# database configs -DB_NAME = defaultdb # example -DB_USER = doadmin # example -DB_PASS = -DB_PORT = -DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com # example - - -# paths -CHROMEDRIVER = /usr/bin/chromedriver -GOOGLECHROME = /usr/bin/google-chrome -CHROMIUM = /usr/bin/chromium - - -# stripe keys -STRIPE_PUBLIC_TEST = -STRIPE_PRIVATE_TEST = -STRIPE_PUBLIC_LIVE = -STRIPE_PRIVATE_LIVE = -STRIPE_ENV = prod - - -# google keys -GOOGLE_CRUX_KEY = - - -# OAuth keys -GOOGLE_OAUTH2_CLIENT_ID = -GOOGLE_OAUTH2_CLIENT_SECRET = - - -# twilio credentials -TWILIO_SID = -TWILIO_AUTH_TOKEN = -TWILIO_NUMBER = - - -# sendgrid configs -SENDGRID_API_KEY = -DEFAULT_TEMPLATE = -DEFAULT_TEMPLATE_NO_BUTTON = -AUTOMATION_TEMPLATE = - - - -# slack credentials -SLACK_APP_ID = -SLACK_CLIENT_ID = -SLACK_CLIENT_SECRET = -SLACK_SIGNING_SECRET = -SLACK_VERIFICATION_TOKEN = -SLACK_BOT_TOKEN = - - -# s3 remote storage credentials -AWS_ACCESS_KEY_ID = -AWS_SECRET_ACCESS_KEY = -AWS_STORAGE_BUCKET_NAME = storage-scanerr # example -AWS_S3_REGION_NAME = sfo3 # example -AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com # example -AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com # example -AWS_LOCATION = static -AWS_DEFAULT_ACL = public-read \ No newline at end of file diff --git a/env/.env.prod.proxy-companion b/env/.env.prod.proxy-companion deleted file mode 100644 index 085d84bf..00000000 --- a/env/.env.prod.proxy-companion +++ /dev/null @@ -1,2 +0,0 @@ -DEFAULT_EMAIL=youremail@yourdomain.com -NGINX_PROXY_CONTAINER=nginx-proxy \ No newline at end of file diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md deleted file mode 100644 index 336cb823..00000000 --- a/k8s/kubernetes-notes.md +++ /dev/null @@ -1,84 +0,0 @@ -### Create k8s files in yaml (kompose) -```shell -kompose convert -f docker-compose.yml -o ./k8s -``` - -### Build k8s -```shell -kubectl apply -f ./k8s/k8s-local.yaml -``` - -### Delete k8s -```shell -kubectl delete -f ./k8s/k8s-local.yaml -``` - -### List containers -```shell -kubectl get pod -``` - -### List pods with IPs -```shell -kubectl get pod -o wide -``` - -### To get all creation events for debugging: -```shell -kubectl get events --sort-by=.metadata.creationTimestamp -``` - -### SSH into container: -```shell -kubectl exec -it celery-849f76858b-bvmqg -- /bin/sh -``` - -### Creating secrets for docker: -```shell -kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com -``` - -#### - Then add this to both celery and app containers: -```yaml -spec: - imagePullSecrets: - - name: regcred -``` - -### Start and Stop minikube -```shell -minikube start -minikube stop -``` - -### Port Forwarding for app -```shell -kubectl port-forward service/app-service 8000:8000 -``` - - - -## Setps to Deploy localy -1. ensure minikube is running - - ``` minikube status ``` -2. create secrets for app image pull from docker - - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com ``` -3. build db-configs-configs - - ``` kubectl apply db-configs.yaml ``` -4. build db-deployment - - ``` kubectl apply db-deployment.yaml ``` -5. build redis-deployment -6. get pod ip of db-deployment - - ``` kubectl get pod --template '{{.status.podIP}}' ``` - - or ``` kubectl get pod -o wide ``` -7. copy ip and paste into app-configs-configs for field "DB_HOST" -8. build app-configs - - ``` kubectl apply app-config.yaml ``` -9. build app-deployment - - ``` kubectl apply db-deployment.yaml ``` -10. build celery-deployment - - ``` kubectl apply db-deployment.yaml ``` -11. port forwarding to app deployment - - ``` kubectl port-forward service/app-service 8000:8000 ``` - - \ No newline at end of file diff --git a/k8s/local/app-configs-example.yaml b/k8s/local/app-configs-example.yaml new file mode 100644 index 00000000..6290e3be --- /dev/null +++ b/k8s/local/app-configs-example.yaml @@ -0,0 +1,80 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-configs +data: + # django + SECRET_KEY : "ask-for-this-or-generate-yourself" + CLIENT_URL_ROOT : "https://app.yourdomain.com" + API_URL_ROOT : "https://api.yourdomain.com" + YELLOWLAB_ROOT : "http://ylt-service" + LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed" + LANDING_API_KEY : "" + LANDING_URL_ROOT : "https://yourdomain.com" + LETSENCRYPT_HOST : "api.yourdomain.com" + VIRTUAL_HOST : "api.yourdomain.com" + VIRTUAL_PORT : "8000" + DJANGO_ALLOWED_HOSTS : "*" + MODE: "local" + DEBUG : "True" + LOCATION : "us" + # admin credentials + ADMIN_USER : "admin_user" + ADMIN_PASS : "f4k3P455w0rd" + ADMIN_EMAIL : "your@email.com" + # email credentials + EMAIL_HOST : "smtp.gmail.com" + EMAIL_PORT : "587" + EMAIL_USE_TLS : "True" + EMAIL_HOST_USER : "your@email.com" + EMAIL_HOST_PASSWORD : "your-email-password" + # database + DB_HOST : "" + DB_NAME : "k8s-pool" + DB_PASS : "" + DB_PORT : "25061" + DB_USER : "" + # paths + CHROMEDRIVER : "/usr/bin/chromedriver" + CHROME_BROWSER : "/usr/bin/chromium" + # stripe keys + STRIPE_PUBLIC_TEST : "pk_test_" + STRIPE_PRIVATE_TEST : "sk_test_" + STRIPE_PUBLIC_LIVE : "pk_live_" + STRIPE_PRIVATE_LIVE : "sk_live_" + STRIPE_ENV : "dev" + # google keys + GOOGLE_CRUX_KEY : "" + # OAuth keys + GOOGLE_OAUTH2_CLIENT_ID : "" + GOOGLE_OAUTH2_CLIENT_SECRET : "" + # twilio credentials + TWILIO_SID : "" + TWILIO_AUTH_TOKEN : "" + TWILIO_NUMBER : "" + # sendgrid configs + SENDGRID_EMAIL: your@email.com + SENDGRID_API_KEY : "" + DEFAULT_TEMPLATE : "" + DEFAULT_TEMPLATE_NO_BUTTON : "" + AUTOMATION_TEMPLATE : "" + # slack credentials + SLACK_APP_ID : "" + SLACK_CLIENT_ID : "" + SLACK_CLIENT_SECRET : "" + SLACK_SIGNING_SECRET : "" + SLACK_VERIFICATION_TOKEN : "" + SLACK_BOT_TOKEN : "" + # s3 remote storage credentials + AWS_ACCESS_KEY_ID : "" + AWS_SECRET_ACCESS_KEY : "" + AWS_STORAGE_BUCKET_NAME : "" + AWS_S3_REGION_NAME : "sfo3" + AWS_S3_ENDPOINT_URL : "https://sfo3.digitaloceanspaces.com" + AWS_S3_URL_PATH : "https://.sfo3.digitaloceanspaces.com" + AWS_LOCATION : "static" + AWS_DEFAULT_ACL : "public-read" + # secrets key (32 bytes) + SECRETS_KEY : "" + # OpenAI API key + GPT_API_KEY : "" diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml index 36be9020..cad462a9 100644 --- a/k8s/local/app-deployment.yaml +++ b/k8s/local/app-deployment.yaml @@ -12,26 +12,13 @@ spec: labels: app: app spec: - imagePullSecrets: - - name: regcred containers: - name: app - image: landonr/scanerr-server + image: cursiondev/server:latest imagePullPolicy: IfNotPresent ports: - containerPort: 8000 - command: - - "sh" - - "-c" - - > - python3 manage.py wait_for_db && - python3 manage.py makemigrations --no-input && - python3 manage.py migrate --no-input && - python3 manage.py collectstatic --no-input && - python3 manage.py create_admin && - python3 manage.py driver_s_test && - python3 manage.py driver_p_test && - python3 manage.py runserver 0.0.0.0:8000 + command: ["/entrypoint.sh", "server", "remote"] envFrom: - configMapRef: name: app-configs diff --git a/k8s/prod/celery-deployment.yaml b/k8s/local/beat-deployment.yaml similarity index 61% rename from k8s/prod/celery-deployment.yaml rename to k8s/local/beat-deployment.yaml index 64ed79e9..b3289bea 100644 --- a/k8s/prod/celery-deployment.yaml +++ b/k8s/local/beat-deployment.yaml @@ -1,24 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: celery-deployment + name: beat-deployment spec: replicas: 1 selector: matchLabels: - app: celery + app: beat template: metadata: labels: - app: celery + app: beat spec: - imagePullSecrets: - - name: regcred containers: - - name: celery - image: landonr/scanerr-server + - name: beat + image: cursiondev/server:latest imagePullPolicy: IfNotPresent - command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"] + command: ["/entrypoint.sh", "beat"] envFrom: - configMapRef: name: app-configs diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml index 64ed79e9..fb38aa1e 100644 --- a/k8s/local/celery-deployment.yaml +++ b/k8s/local/celery-deployment.yaml @@ -12,20 +12,18 @@ spec: labels: app: celery spec: - imagePullSecrets: - - name: regcred containers: - name: celery - image: landonr/scanerr-server + image: cursiondev/server:latest imagePullPolicy: IfNotPresent - command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"] + command: ["/entrypoint.sh", "celery"] envFrom: - configMapRef: name: app-configs resources: limits: cpu: "1" - memory: "1Gi" + memory: "2Gi" requests: cpu: "500m" memory: "500Mi" diff --git a/k8s/local/db-deployment.yaml b/k8s/local/db-deployment.yaml index beee6ddd..f5c82eae 100644 --- a/k8s/local/db-deployment.yaml +++ b/k8s/local/db-deployment.yaml @@ -14,12 +14,12 @@ spec: spec: containers: - name: db - image: postgres:10-alpine + image: postgres:14-alpine ports: - containerPort: 5432 envFrom: - configMapRef: - name: db-configs + name: app-configs volumeMounts: - name: pgdata-volume mountPath: /var/lib/postgresql/data diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml new file mode 100644 index 00000000..9c24665d --- /dev/null +++ b/k8s/prod/app-cert-issuer.yaml @@ -0,0 +1,19 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-nginx + namespace: default +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory # https://acme-staging-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: hello@cursion.dev + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-nginx-private-key + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + class: nginx diff --git a/k8s/prod/app-configs-example.yaml b/k8s/prod/app-configs-example.yaml new file mode 100644 index 00000000..e6f23e2f --- /dev/null +++ b/k8s/prod/app-configs-example.yaml @@ -0,0 +1,80 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-configs +data: + # django + SECRET_KEY : "ask-for-this-or-generate-yourself" + CLIENT_URL_ROOT : "https://app.yourdomain.com" + API_URL_ROOT : "https://api.yourdomain.com" + MCP_URL_ROOT : "https://mcp.yourdomain.com" + YELLOWLAB_ROOT : "http://ylt-service" + LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed" + LANDING_API_KEY : "" + LANDING_URL_ROOT : "https://yourdomain.com" + LETSENCRYPT_HOST : "api.yourdomain.com" + VIRTUAL_HOST : "api.yourdomain.com" + VIRTUAL_PORT : "8000" + DJANGO_ALLOWED_HOSTS : "*" + MODE: "local" + DEBUG : "True" + LOCATION : "us" + # admin credentials + ADMIN_USER : "admin_user" + ADMIN_PASS : "f4k3P455w0rd" + ADMIN_EMAIL : "your@email.com" + # email credentials + EMAIL_HOST : "smtp.gmail.com" + EMAIL_PORT : "587" + EMAIL_USE_TLS : "True" + EMAIL_HOST_USER : "your@email.com" + EMAIL_HOST_PASSWORD : "your-email-password" + # database + DB_HOST : "" + DB_NAME : "k8s-pool" + DB_PASS : "" + DB_PORT : "25061" + DB_USER : "" + # paths + CHROMEDRIVER : "/usr/bin/chromedriver" + CHROME_BROWSER : "/usr/bin/chromium" + # stripe keys + STRIPE_PUBLIC_TEST : "pk_test_" + STRIPE_PRIVATE_TEST : "sk_test_" + STRIPE_PUBLIC_LIVE : "pk_live_" + STRIPE_PRIVATE_LIVE : "sk_live_" + STRIPE_ENV : "dev" + # google keys + GOOGLE_CRUX_KEY : "" + # OAuth keys + GOOGLE_OAUTH2_CLIENT_ID : "" + GOOGLE_OAUTH2_CLIENT_SECRET : "" + # twilio credentials + TWILIO_SID : "" + TWILIO_AUTH_TOKEN : "" + TWILIO_NUMBER : "+" + # sendgrid configs + SENDGRID_API_KEY : "" + DEFAULT_TEMPLATE : "" + DEFAULT_TEMPLATE_NO_BUTTON : "" + AUTOMATION_TEMPLATE : "" + # slack credentials + SLACK_APP_ID : "" + SLACK_CLIENT_ID : "" + SLACK_CLIENT_SECRET : "" + SLACK_SIGNING_SECRET : "" + SLACK_VERIFICATION_TOKEN : "" + SLACK_BOT_TOKEN : "" + # s3 remote storage credentials + AWS_ACCESS_KEY_ID : "" + AWS_SECRET_ACCESS_KEY : "" + AWS_STORAGE_BUCKET_NAME : "" + AWS_S3_REGION_NAME : "sfo3" + AWS_S3_ENDPOINT_URL : "https://sfo3.digitaloceanspaces.com" + AWS_S3_URL_PATH : "https://.sfo3.digitaloceanspaces.com" + AWS_LOCATION : "static" + AWS_DEFAULT_ACL : "public-read" + # secrets key (32 bytes) + SECRETS_KEY : "" + # OpenAI API key + GPT_API_KEY : "" diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml index c91eb55a..862c9c40 100644 --- a/k8s/prod/app-deployment.yaml +++ b/k8s/prod/app-deployment.yaml @@ -2,58 +2,56 @@ apiVersion: apps/v1 kind: Deployment metadata: name: app-deployment + labels: + deployment: app spec: - replicas: 1 + replicas: 2 selector: matchLabels: - app: app + app: app-deployment template: metadata: labels: - app: app + app: app-deployment spec: imagePullSecrets: - name: regcred containers: - - name: app - image: landonr/scanerr-server + - name: cursion-server + image: # cursiondev/server:8e56596 # imagePullPolicy: IfNotPresent ports: - containerPort: 8000 - command: - - "sh" - - "-c" - - > - python3 manage.py wait_for_db && - python3 manage.py makemigrations --no-input && - python3 manage.py migrate --no-input && - python3 manage.py collectstatic --no-input && - python3 manage.py create_admin && - python3 manage.py driver_s_test && - python3 manage.py driver_p_test && - gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000" + command: ["/entrypoint.sh", "server", "remote"] envFrom: - configMapRef: name: app-configs + env: + - name: THIS_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name resources: limits: cpu: "1" - memory: "1Gi" + memory: "2.5Gi" requests: - cpu: "500m" - memory: "500Mi" - + cpu: "0.75" + memory: "1.5Gi" --- apiVersion: v1 kind: Service metadata: name: app-service + labels: + service: app spec: + # type: NodePort + # externalTrafficPolicy: Cluster selector: - app: app + app: app-deployment ports: - - protocol: TCP - port: 8000 - targetPort: 8000 - type: NodePort + - name: http + port: 80 + targetPort: 8000 diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml new file mode 100644 index 00000000..45442731 --- /dev/null +++ b/k8s/prod/app-ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: app-ingress + annotations: + kubernetes.io/ingress.class: nginx + ## ensure below section is commented out on first creation ## + # cert-manager.io/cluster-issuer: letsencrypt-nginx +spec: + ## ensure below section is comented out on first creation ## + # tls: + # - hosts: + # - api.cursion.dev + # secretName: letsencrypt-nginx + rules: + - host: api.cursion.dev + http: + paths: + - backend: + service: + name: app-service + port: + number: 80 + path: / + pathType: Prefix + ingressClassName: nginx \ No newline at end of file diff --git a/k8s/prod/app-loadbalancer.yaml b/k8s/prod/app-loadbalancer.yaml new file mode 100644 index 00000000..99115af2 --- /dev/null +++ b/k8s/prod/app-loadbalancer.yaml @@ -0,0 +1,688 @@ + +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + +--- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +automountServiceAccountToken: true +--- +# Source: ingress-nginx/templates/controller-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +data: + allow-snippet-annotations: 'true' + use-proxy-protocol: 'true' +--- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + appProtocol: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: 'true' + service.beta.kubernetes.io/do-loadbalancer-hostname: "api.cursion.dev" + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + appProtocol: http + - name: https + port: 443 + protocol: TCP + targetPort: https + appProtocol: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + spec: + dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --controller-class=k8s.io/ingress-nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 300 + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +# Source: ingress-nginx/templates/controller-ingressclass.yaml +# We don't support namespaced ingressClass yet +# So a ClusterRole and a ClusterRoleBinding is required +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: nginx + namespace: ingress-nginx +spec: + controller: k8s.io/ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + name: ingress-nginx-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1/ingresses + timeoutSeconds: 29 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + namespace: ingress-nginx + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-4.0.15 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.1.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 2000 \ No newline at end of file diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml new file mode 100644 index 00000000..45242d5f --- /dev/null +++ b/k8s/prod/beat-deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beat-deployment + labels: + deployment: beat +spec: + replicas: 1 + selector: + matchLabels: + app: beat-deployment + strategy: {} + template: + metadata: + labels: + app: beat-deployment + spec: + imagePullSecrets: + - name: regcred + containers: + - name: beat + image: # cursiondev/server:8e56596 # + imagePullPolicy: Always + command: ["/entrypoint.sh", "beat"] + envFrom: + - configMapRef: + name: app-configs + resources: + limits: + cpu: ".5" + memory: "1Gi" + requests: + cpu: ".5" + memory: "1Gi" + + diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml new file mode 100644 index 00000000..520e3a37 --- /dev/null +++ b/k8s/prod/celery-autoscaler.yaml @@ -0,0 +1,18 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: celery-scaler +spec: + scaleTargetRef: + name: celery-scheduled-deployment + cooldownPeriod: 300 + pollingInterval: 15 + minReplicaCount: 2 + maxReplicaCount: 15 + triggers: + - type: metrics-api + metadata: + targetValue: "10" + url: "https://api.cursion.dev/v1/ops/metrics/celery?queue=scheduled" + valueLocation: "working_len" + timeout: "20000" diff --git a/k8s/prod/celery-on-demand-deployment.yaml b/k8s/prod/celery-on-demand-deployment.yaml new file mode 100644 index 00000000..2429dcb6 --- /dev/null +++ b/k8s/prod/celery-on-demand-deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-on-demand-deployment + labels: + deployment: celery-on-demand +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 50% + maxUnavailable: 50% + selector: + matchLabels: + app: celery-on-demand-deployment + template: + metadata: + labels: + app: celery-on-demand-deployment + spec: + terminationGracePeriodSeconds: 300 + imagePullSecrets: + - name: regcred + containers: + - name: celery-on-demand + image: + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh", "celery", "on_demand"] + envFrom: + - configMapRef: + name: app-configs + env: + - name: THIS_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # remove below if not using NAT gateway + - name: HTTP_PROXY + value: "http://10.124.0.29:8888" + - name: HTTPS_PROXY + value: "http://10.124.0.29:8888" + - name: NO_PROXY + value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8,.googleapis.com,.google.com,clients2.google.com,mtalk.google.com" + resources: + limits: + cpu: "1" + memory: "2.5Gi" + requests: + cpu: "0.75" + memory: "1.5Gi" + lifecycle: + preStop: + exec: + command: ["python3", "manage.py", "terminate_worker"] + livenessProbe: + exec: + command: ["/healthcheck.sh", "celery"] + initialDelaySeconds: 120 + periodSeconds: 60 + failureThreshold: 12 + readinessProbe: + exec: + command: ["/healthcheck.sh", "celery"] + initialDelaySeconds: 120 + periodSeconds: 60 + failureThreshold: 12 + diff --git a/k8s/prod/celery-scheduled-deployment.yaml b/k8s/prod/celery-scheduled-deployment.yaml new file mode 100644 index 00000000..0f02c070 --- /dev/null +++ b/k8s/prod/celery-scheduled-deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-scheduled-deployment + labels: + deployment: celery +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 50% + maxUnavailable: 50% + selector: + matchLabels: + app: celery-scheduled-deployment + template: + metadata: + labels: + app: celery-scheduled-deployment + spec: + terminationGracePeriodSeconds: 300 + imagePullSecrets: + - name: regcred + containers: + - name: celery + image: # cursiondev/server:65ccd89 # + imagePullPolicy: IfNotPresent + command: ["/entrypoint.sh", "celery", "scheduled"] + envFrom: + - configMapRef: + name: app-configs + env: + - name: THIS_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # remove below if not using NAT gateway + - name: HTTP_PROXY + value: "http://10.124.0.29:8888" + - name: HTTPS_PROXY + value: "http://10.124.0.29:8888" + - name: NO_PROXY + value: "localhost,127.0.0.1,.svc.cluster.local,10.0.0.0/8,.googleapis.com,.google.com,clients2.google.com,mtalk.google.com" + # resources: # -> large node pool (4vcpu & 8Gi) + # limits: + # cpu: "2" + # memory: "6Gi" + # requests: + # cpu: "1" + # memory: "4Gi" + resources: # -> small node pool (2vcpu & 4Gi) + limits: + cpu: "1" + memory: "2.5Gi" + requests: + cpu: "0.75" + memory: "1.5Gi" + lifecycle: + preStop: + exec: + command: ["python3", "manage.py", "terminate_worker"] + livenessProbe: + exec: + command: ["/healthcheck.sh", "celery"] + initialDelaySeconds: 120 + periodSeconds: 60 + failureThreshold: 12 + readinessProbe: + exec: + command: ["/healthcheck.sh", "celery"] + initialDelaySeconds: 120 + periodSeconds: 60 + failureThreshold: 12 diff --git a/k8s/prod/old_configs/app-pvc.yaml b/k8s/prod/old_configs/app-pvc.yaml new file mode 100644 index 00000000..baa2b84c --- /dev/null +++ b/k8s/prod/old_configs/app-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: app-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/k8s/prod/old_configs/celery-autoscaler.yaml b/k8s/prod/old_configs/celery-autoscaler.yaml new file mode 100644 index 00000000..a7e31e77 --- /dev/null +++ b/k8s/prod/old_configs/celery-autoscaler.yaml @@ -0,0 +1,43 @@ +# apiVersion: keda.sh/v1alpha1 +# kind: ScaledObject +# metadata: +# name: celery-scaler +# spec: +# scaleTargetRef: +# name: celery-deployment +# pollingInterval: 3 +# minReplicaCount: 2 +# maxReplicaCount: 15 +# triggers: +# - type: redis +# metadata: +# address: redis.default.svc.cluster.local:6379 # Format must be host:port redis:6379 +# listName: celery # Required +# listLength: "5" # Required +# activationListLength: "5" # optional +# enableTLS: "false" # optional +# unsafeSsl: "false" # optional +# databaseIndex: "0" # optional + + + + +# apiVersion: keda.sh/v1alpha1 +# kind: ScaledObject +# metadata: +# name: celery-scaler +# spec: +# scaleTargetRef: +# name: celery-deployment +# cooldownPeriod: 4000 +# pollingInterval: 3 +# minReplicaCount: 2 +# maxReplicaCount: 15 +# triggers: +# - type: rabbitmq +# metadata: +# host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication. +# mode: QueueLength # QueueLength or MessageRate +# value: "5" # message backlog or publish/sec. target per instance +# activationValue: "5" # Optional. Activation threshold +# queueName: celery \ No newline at end of file diff --git a/k8s/prod/old_configs/rabbitmq-deployment.yaml b/k8s/prod/old_configs/rabbitmq-deployment.yaml new file mode 100644 index 00000000..18ada4c6 --- /dev/null +++ b/k8s/prod/old_configs/rabbitmq-deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + service: rabbitmq + name: rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + service: rabbitmq + strategy: {} + template: + metadata: + labels: + service: rabbitmq + spec: + restartPolicy: Always + containers: + - image: rabbitmq:alpine + name: rabbitmq + ports: + - containerPort: 5672 + resources: + limits: + cpu: "250m" + memory: "250Mi" + requests: + cpu: "100m" + memory: "100Mi" +status: {} + + + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + service: rabbitmq + name: rabbitmq +spec: + ports: + - name: "5672" + port: 5672 + targetPort: 5672 + selector: + service: rabbitmq diff --git a/k8s/prod/tasks-cronjob.yml b/k8s/prod/tasks-cronjob.yml new file mode 100644 index 00000000..24faeaf3 --- /dev/null +++ b/k8s/prod/tasks-cronjob.yml @@ -0,0 +1,20 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: tasks-cronjob +spec: + schedule: "*/5 * * * *" # every 5 minutes + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: retry-tasks + image: curlimages/curl:latest + args: + - /bin/sh + - -c + - curl -X GET https://api.cursion.dev/v1/ops/tasks/retry + restartPolicy: OnFailure diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml new file mode 100644 index 00000000..e7490c5c --- /dev/null +++ b/k8s/prod/ylt-autoscaler.yaml @@ -0,0 +1,17 @@ +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: ylt-scaler +spec: + scaleTargetRef: + name: ylt-deployment + cooldownPeriod: 300 + pollingInterval: 15 + minReplicaCount: 2 + maxReplicaCount: 7 + triggers: + - type: metrics-api + metadata: + targetValue: "10" + url: "https://api.cursion.dev/v1/ops/metrics/celery" + valueLocation: "working_len" \ No newline at end of file diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml new file mode 100644 index 00000000..b2899d8f --- /dev/null +++ b/k8s/prod/ylt-deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ylt-deployment + labels: + deployment: yellowlab +spec: + replicas: 2 + selector: + matchLabels: + app: ylt-deployment + template: + metadata: + labels: + app: ylt-deployment + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: yellowlab + image: cursiondev/ylt + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: 8383 + securityContext: + privileged: true + resources: + limits: + cpu: "1" + memory: "4Gi" + requests: + cpu: ".5" + memory: "1Gi" +--- + +apiVersion: v1 +kind: Service +metadata: + name: ylt-service + labels: + service: ylt +spec: + selector: + app: ylt-deployment + ports: + - name: http + port: 80 + targetPort: 8383 \ No newline at end of file diff --git a/legal/COMMERCIAL.md b/legal/COMMERCIAL.md new file mode 100644 index 00000000..ddecc473 --- /dev/null +++ b/legal/COMMERCIAL.md @@ -0,0 +1,103 @@ +Copyright (C) 2026 Cursion + + +Cursion Commercial Software License Terms + +1. Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order. +(i) Software: Cursion +(ii) Developer: Grey Labs, LLC (https://greylabs.io) +2. Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues. +3. Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software. +4. Billing. +(a) Bills, Fees, and Payment. The vendor agrees to bill the customer per the order. The customer agrees to pay the fees on the order, using the payment method on the order. +(b) Billing Errors. The customer agrees to give the vendor notice of any suspected error on a bill before the deadline for payment. Both sides agree to resolve any concerns about bill accuracy promptly and in good faith. The customer agrees to pay the undisputed part of each bill by the original deadline, and any part of the bill resolved later within seven days of resolution. +5. Term and Termination. +(a) Perpetual. This agreement continues until one side or the other ends it. +(b) Termination. Either side can terminate this agreement immediately if the other side breaches and fails to cure their breach within fourteen days of notice. +6. Use. +(a) Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates. +(b) Prohibited Uses. The customer may not: +(i) sell, lease, license, or sublicense the software or documentation +(ii) allow access to the software by others not licensed under this agreement +(iii) share copies of the software or documentation with with others not licensed under this agreement +(iv) make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers +(v) assist or allow others to use the software against the terms of this agreement +7. Licenses. +(a) Software Copyright License. The vendor grants the customer and each authorized user a standard license for any copyrights in the software that the vendor can license, to copy, install, back up, and use the software as allowed under this agreement. +(b) Software Patent License. The vendor grants the customer and each authorized user a standard license for any patents the vendor can license or becomes able to license, to use the software as allowed under this agreement. +(c) Documentation Copyright License. The vendor grants the customer and each authorized user a standard license for any copyrights in the documentation that the vendor can license, to read, back up, and copy the documentation. +(d) Standard License Terms. A standard license means a nonexclusive license for the term of this agreement, for versions of the software covered by this agreement, that is conditional on payment of all fees as required by this agreement and subject to any use limits in this agreement. +(e) No Other Licenses. Apart from the licenses in Section 7 (Licenses), this agreement does not license or assign any intellectual property rights. +8. Open Source. +(a) Open Source Compliance. Some components of the software may be open source software available under free, public licenses. If the public license terms for any open source component conflict with the terms of this agreement, only the public license terms apply to that component, not the terms of this agreement. If the license terms for any open source component require an offer of source code or other information related to that component, the vendor agrees to provide on written request. +(b) Dual Licensing. If any part of the software is or becomes available under a public license: +(i) While the customer’s licenses continue, the customer and each authorized user must abide by this agreement, not the public license. +(ii) The customer must abide by the terms of the public license for any versions of the software not covered by this agreement. +9. Delivery. +(a) Materials. The vendor agrees to deliver the following to the customer within three days: +(i) a copy of the software’s source code in the preferred form for making changes +(ii) copies of any scripts or configuration files necessary to compile the software’s source code +(iii) a copy of the software’s documentation +(b) Method. The vendor agrees to deliver all materials by e-mail or by making them available to download online, without any additional charge. The vendor agrees to make new versions of the software covered by this agreement available in the same way, within three days of making it generally available. +(c) License Keys. If the software requires license keys to function, the vendor agrees to give the customer those keys by e-mail within three days. If license keys for the software expire over time, the vendor agrees to give the customer new license keys by e-mail at least two weeks before the last keys expire. The customer agrees to share license keys only as required for use of the software as allowed under this this agreement, and to secure its license keys at least as well as its confidential business information. +10. Technical Support. +(a) Basic Support. During its regular business hours, the vendor agrees to respond to e-mail support requests from customer or any authorized user about configuration of, use of, and problems with the software and its documentation. The vendor does not agree to any specific service levels for response to support requests. +(b) Access. On the vendor’s request, the customer agrees to give the vendor prompt access to personnel, systems, and information needed to respond to support requests. +(c) Confidentiality. On the customer’s request, the vendor will agree to the terms of a standard, published, mutual nondisclosure agreement with the customer, for the purpose of fulfilling its support obligations under this agreement. +11. Warranties. +(a) Perform As Documented. The vendor guarantees that the software will perform as described in its documentation while this agreement continues, except when: +(i) using older version of the software than the latest provided under this agreement +(ii) using the software with modifications +(iii) running the software using hardware or software different from that required, according to the documentation +(iv) combining the software with other software or hardware in ways not described in the documentation +(b) Malware. The vendor guarantees that the software it delivers will be free of malicious code, such as computer worms and viruses. +(c) Limiting Code. The developer guarantees that the software it delivers will be free of code that automatically limits or disables software functionality, other than: +(i) code that limits or disables functionality on failure to validate license keys +(ii) code that limits or disables functionality based on automatic monitoring of agreed limits on usage +(d) Software Dependencies. If the software depends on, installs, configures, or links to other software in order to function, the vendor guarantees that those software dependencies will be either provided in the copies of the software delivered to the customer or generally available for the customer to download, free or charge, from a well known website or Internet service, such as an open source software package repository. +12. Liability. +(a) Disclaimer. Section 11 (Warranties) sets out the only warranties the vendor provides for the software. The vendor disclaims any warranties the law might otherwise imply, like warranties of merchantability, fitness for any particular purpose, title, or noninfringement. +(b) Unforeseeable Damages. Neither side will be liable for breach-of-contract damages they could not have reasonably foreseen when entering into this agreement. +(c) Liability Cap. Except for Section 12(d) (Uncapped Liabilities), neither side’s total liability for breach of this agreement will exceed the amount of fees the vendor received from the customer under this agreement during the twelve months before the first claim is made. This limit applies even if the side liable is advised that the other may suffer damages, and even if the customer paid no fees at all. +(d) Uncapped Liabilities. Section 12(c) (Liability Cap) does not apply to: +(i) the customer’s obligations to pay fees +(ii) the vendor’s obligations to indemnify the customer +(iii) liabilities the law requires to be unlimited +13. Indemnities. These indemnities apply as long as the customer has paid all licensing fees as required by this agreement: +(a) General Indemnity. Subject to Section 13(e) (Indemnification Process), the vendor agrees to indemnify the customer for legal claims by others alleging that the software infringes any copyright, trademark, or trade secret right, or breaks any law. +(b) Patent Indemnity. The vendor will not indemnify the customer for any claims by others alleging that the software infringes any patent. +(c) Scope of Indemnity. Throughout this agreement, to indemnify means to indemnify and hold the customer and its personnel harmless for all liability, expenses, damages, and costs, as well as to defend the indemnified party. +(d) Only Remedy. Both sides agree that indemnification will be the only legal remedy for claims covered by indemnity. +(e) Indemnification Process. Both sides agree that to receive indemnification under this agreement, they must give notice of any covered claim quickly, allow the other side to control investigation, defense, and settlement, and cooperate with those efforts. Both sides agree that if they fail to give notice of any covered claim quickly, indemnification will not cover amounts that could have been defended against or mitigated if notice had been given quickly. Both sides agree that if they take control of the defense and settlement of any covered claim, they will not agree to any settlements that admit fault or impose obligations on the other side without their signed, written permission. +(f) Repair, Replace, Refund. If the vendor or the customer receives written notice of a claim that the software infringes any intellectual property right or breaks any law, or vendor reasonably anticipates a claim of that kind: +(i) The developer may provide the customer a new version of the software that no longer infringes or breaks the law. That new version will be covered by this agreement. The customer will not pay any additional fee for the new version. +(ii) If the problem is infringement, the developer may get licenses for the customer so that the customer’s use of the software no longer infringes. +(iii) If the problem is illegality, the developer may get the approvals, licenses, or other requirements needed to abide by the law. +(iv) The developer may refund any fees the customer has prepaid under this agreement for time remaining in the term of this agreement, on a proportional basis, and end this agreement immediately by giving the customer notice. +14. Tax. +(a) Taxes on Fees. The customer agrees to pay all tax on fees under this agreement, except tax on the vendor’s income. +(b) Tax Withholding. If the customer is located outside the United States and local law requires the customer to withhold taxes on fees paid under this agreement: +(i) The customer agrees to make the required tax withholding payments for the vendor by deducting the right amounts from payments to the vendor and paying them to the proper tax authorities. +(ii) The customer agrees to increase the amount of each payment made under this agreement, to offset withholding, so that the vendor receives the full amount owed. +(iii) The customer agrees to give the vendor relevant official tax documentation and tax receipts showing that withholding was required and that proper withholding payment was made, as soon as possible after making any withholding payment. +15. General Contract Terms. +(a) Notices. Both sides agree to give notice under this agreement, the side giving notice must send by e-mail to the address the recipient gave with its signature, or to a different address given later for notices going forward, in the English language. If either side finds that e-mail can’t be delivered to the e-mail address given, the sender may give notice by registered mail to the address on file for the recipient with the state under whose laws it is organized. +(b) Governing Law. This agreement will be governed by the law of the jurisdiction of the address the vendor gives with its signature. +(c) No CISG. The United Nations Convention on Contracts for the International Sale of Goods will not apply to this agreement. +(d) No UCITA. As far as the law allows, the Uniform Computer Information Transactions Act will not apply to this agreement. +(e) Dispute Resolution. Any dispute, controversy or claim arising out of or relating to this contract, including the formation, interpretation, breach or termination thereof, including whether the claims asserted are arbitrable, will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The Tribunal will consist of one arbitrator. The place of arbitration will be the capital of the jurisdiction whose laws govern this agreement. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator(s) may be entered in any court having jurisdiction thereof. +(f) Enforcement. Only the parties may enforce rights under this agreement. +(g) Forum for Disputes. Both sides agree to bring any lawsuits related to this agreement in courts in the capital of the jurisdiction whose laws govern this agreement. Both sides consent to the exclusive jurisdiction of those courts and waive any objection that they would be an inconvenient forum for a lawsuit. Both sides agree that the other side can enforce judgments from those courts in other jurisdictions. +(h) Only Terms. Both sides intend the terms of this agreement, together with the order, as the final, complete, and only expression of their agreement about the software. +(i) Unenforceable Terms. If a court decides that any part of this agreement is invalid or unenforceable for any reason, and that enforcing the rest of this agreement would not defeat the purpose of this agreement, then rest of this agreement will still apply. +(j) Excuses. Neither side will be liable for any failure or delay meeting any obligation under this agreement caused by: +(i) failure of the other side or its personnel to meet their obligations under this agreement +(ii) actions done or delayed at the written request of the other side +(iii) fire, flood, earthquake, and other natural disasters +(iv) declared and undeclared wars, acts of terrorism, sabotage, riots, civil disorder, rebellions, and revolutions +(v) extraordinary malfunction of Internet infrastructure, data centers, or communication utilities +(vi) government actions taken in response to any of these causes +(k) Amendments. Both sides may change or add to the terms of this agreement only by signing a written amendment. +(l) Waivers. Both sides will waive terms of this agreement, if at all, only in signed writing. +(m) No Assignment. Neither side may assign any right under this agreement without the other side’s signed, written permission. Neither side will unreasonably refuse permission. Any attempt to assign against the terms of this agreement will have no legal effect. +(n) No Delegation. Neither side may delegate any performance under this agreement. Any attempt to delegate will have no legal effect. diff --git a/legal/OSS.md b/legal/OSS.md new file mode 100644 index 00000000..bf4cd625 --- /dev/null +++ b/legal/OSS.md @@ -0,0 +1,171 @@ +GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. + +The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. + +To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. + +For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. + +Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. + +For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. + +Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. + +Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. + +The precise terms and conditions for copying, distribution and modification follow. + +TERMS AND CONDITIONS 0. Definitions. “This License” refers to version 3 of the GNU General Public License. + +“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. + +“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations. + +To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work. + +A “covered work” means either the unmodified Program or a work based on the Program. + +To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. + +To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. + +1. Source Code. The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work. + +A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. + +The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. + +The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. + +4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: + +a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”. c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. + +6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: + +a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. + +A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. + +“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. + +7. Additional Terms. “Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: + +a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. + +8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. + +9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. + +An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. + +11. Patents. A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”. + +A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. + +In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. + +A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. + +13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. + +14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. + +15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found. + +Copyright (C) + +This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. + +If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: + +Copyright (C) + +This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”. + +You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . + +The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . \ No newline at end of file diff --git a/nginx/README.md b/nginx/README.md new file mode 100644 index 00000000..60b38868 --- /dev/null +++ b/nginx/README.md @@ -0,0 +1,4 @@ +## Build & Push Instructions +1. Ensure you are in the root of `nginx` +2. Build Dockerfile `docker build --platform linux/amd64 . -t 'cursiondev/nginx:latest'` +3. Push to dock Dockerfile `docker push cursiondev/nginx:latest` \ No newline at end of file diff --git a/nginx/vhost.d/default b/nginx/vhost.d/default index c498447b..fe37f6ef 100644 --- a/nginx/vhost.d/default +++ b/nginx/vhost.d/default @@ -1,9 +1,8 @@ -location /static/ { - alias /app/static/; +location /staticfiles/ { + alias /app/staticfiles/; add_header Access-Control-Allow-Origin *; } - diff --git a/notes/Docker.md b/notes/Docker.md new file mode 100644 index 00000000..e3622179 --- /dev/null +++ b/notes/Docker.md @@ -0,0 +1,33 @@ +# Docker Guide + + +Install and run locally on your machine using `Docker`. + +> Ensure you have `Docker` and `Docker-desktop` installed and running on your machine prior to begining this guide. + +  + + +### 1. **Clone the repo** +```shell +git clone https://github.com/cursion-dev/server.git +``` + + +### 2. **Export `CURSION_ROOT`** +```shell +echo 'export CURSION_ROOT=' >> ~/.zshrc # (or ~/.bash_profile) +``` + + +### 3. **Update `.env.local`** +Prior to running the app, be sure to update `.env.local.example` with your unique values, and remove the `.example` extention from the file. + + +### 4. **Build and Run** +```shell +source ./setup/scripts/local.sh +``` +  + + diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md new file mode 100644 index 00000000..b826c783 --- /dev/null +++ b/notes/Kubernetes.md @@ -0,0 +1,102 @@ +# Kubernetes Guide + + +Install and run locally on your machine using `Kubernetes`. + +> Ensure you have `Docker` and `minikube` installed and running on your machine prior to this step. + +  + + +### 1. **Clone the repo** +```shell +$ git clone https://github.com/cursion-dev/server.git +``` + + +### 2. **Export `CURSION_ROOT`** +```shell +echo 'export CURSION_ROOT=' >> ~/.zshrc # (or ~/.bash_profile) +``` + + +### 3. Ensure minikube is running +```shell +minikube status +``` + + +### 4. **Update config map** +Prior to running the app, be sure to update `app-configs-example.yaml` with your unique values, and remove the trailing `-example` string from the file. + + +#### 4.1 Create regcred for Docker Hub +``` shell +kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username= --docker-password= --docker-email= +``` + + +### 5. Apply app-configs +```shell +kubectl apply $CURSION_ROOT/k8s/local/app-configs.yaml +``` + + +### 6. Apply db-deployment +```shell +kubectl apply $CURSION_ROOT/k8s/local/db-deployment.yaml +``` + + +### 7. Apply redis-deployment +```shell +kubectl apply $CURSION_ROOT/k8s/local/redis-deployment.yaml +``` + + +### 8. Get pod ip of db-deployment +```shell +kubectl get pod -o wide +``` + + +### 9. Paste db pod IP into app-configs for field "DB_HOST" +```shell +kubectl apply $CURSION_ROOT/k8s/local/app-config.yaml +``` + + +### 10. Apply app-deployment +```shell +kubectl apply $CURSION_ROOT/k8s/local/app-deployment.yaml +``` + + +### 11. Apply celery-deployment +```shell +kubectl apply celery-deployment.yaml +``` + + +### 12. Forward Port `8000` to app +```shell +kubectl port-forward service/app-service 8000:8000 +``` + +---- + +  + +## Update to New Version + +> Ensure container version tags are up-to-date in each `.yaml` deployment file. + +#### 1. Apply changes +```shell +kubectl apply -f app-deployment.yaml,celery-deployment.yaml,beat-deployment.yaml +``` + +#### 2. Restart deployments +```shell +kubectl rollout restart deployment app-deployment celery-deployment beat-deployment +``` \ No newline at end of file diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt new file mode 100644 index 00000000..96a53ac8 --- /dev/null +++ b/setup/requirements/requirements.txt @@ -0,0 +1,61 @@ +amqp==5.2.0 +asgiref==3.8.1 +beautifulsoup4==4.12.2 +billiard==4.2.0 +boto3==1.40.52 +celery==5.4.0 +certifi==2025.10.5 +chardet==4.0.0 +click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +cryptography==43.0.3 +Django==5.0.6 +django-celery-beat==2.7.0 +django-filter==24.2 +djangorestframework==3.15.1 +django-markdownify==0.9.5 +django-cors-headers==4.3.1 +django-storages==1.14.3 +djangorestframework-simplejwt==5.3.1 +docker==5.0.0 +gunicorn==20.1.0 +humanize==3.7.0 +idna==2.10 +imutils==0.5.4 +kombu==5.3.7 +Markdown==3.6 +numpy~=1.26.4 +openai==2.8.1 +opencv-python==4.5.5.64 +Pillow==10.3.0 +prometheus-client==0.8.0 +prompt-toolkit==3.0.43 +psycopg2==2.9.9 +pydantic==2.10.6 +pyjwt==2.1.0 +pytz==2021.1 +redis==3.5.3 +requests==2.32.5 +reportlab==4.2.0 +scikit-image==0.23.2 +scipy==1.13.0 +selenium==4.36.0 +sendgrid==6.9.7 +setuptools==75.1.0 +six==1.16.0 +slack-sdk==3.11.2 +sqlparse==0.4.1 +stripe==11.4.1 +tiktoken==0.9.0 +tornado==6.1 +twilio==7.3.0 +urllib3==2.5.0 +vine==5.1.0 +wcwidth==0.2.5 +websocket-client==1.9.0 +whitenoise==6.1.0 + + + diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh new file mode 100755 index 00000000..9e6f6444 --- /dev/null +++ b/setup/scripts/entrypoint.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# /entrypoint.sh + + +# spin up server in local, remote, or stage env +if [[ $1 == *"server"* ]] + then + if [[ $2 == *"local"* ]] + then + python3 manage.py wait_for_db && + python3 manage.py migrate --no-input && + python3 manage.py create_admin && + python3 manage.py verify_account && + python3 manage.py create_tasks && + python3 manage.py test_driver && + python3 manage.py runserver 0.0.0.0:8000 + fi + if [[ $2 == *"remote"* ]] + then + python3 manage.py wait_for_db && + python3 manage.py migrate --no-input && + python3 manage.py create_admin && + python3 manage.py verify_account && + python3 manage.py create_tasks && + python3 manage.py test_driver && + gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug cursion.wsgi:application --bind 0.0.0.0:8000 + fi + if [[ $2 == *"stage"* ]] + then + python3 manage.py wait_for_db && + python3 manage.py makemigrations --no-input && + python3 manage.py migrate --no-input + fi +fi + +# spin up celery +if [[ $1 == *"celery"* ]] + then + python3 manage.py wait_for_db && + echo "pausing for migrations to complete..." && sleep 7s && + QUEUES=${2:-"scheduled,on_demand"} + CONCURRENCY=${3:-${CELERY_CONCURRENCY:-""}} + EXTRA_ARGS="" + if [[ -n "$CONCURRENCY" ]]; then + EXTRA_ARGS="--concurrency=$CONCURRENCY" + fi + celery -A cursion worker -E --loglevel=info -O fair --hostname=celery@$(hostname) -Q "$QUEUES" $EXTRA_ARGS +fi + +# spin up celery beat +if [[ $1 == *"beat"* ]] + then + python3 manage.py wait_for_db && + echo "pausing for migrations to complete..." && sleep 7s && + celery -A cursion beat --scheduler django --loglevel=info +fi diff --git a/setup/scripts/healthcheck.sh b/setup/scripts/healthcheck.sh new file mode 100755 index 00000000..980aeb7c --- /dev/null +++ b/setup/scripts/healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# /healthcheck.sh + + +# check celery worker +if [[ $1 == *"celery"* ]] + then + celery -A cursion inspect ping -d "celery@$(hostname)" --timeout=15 | grep -q OK +fi \ No newline at end of file diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh new file mode 100644 index 00000000..8d822ade --- /dev/null +++ b/setup/scripts/local.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# ensure you create $CURSION_ROOT first: +# " echo 'export CURSION_ROOT=' >> ~/.zshrc (or ~/.bash_profile) " + +cd $CURSION_ROOT && +{ + docker compose -f docker-compose.yml down && + docker volume rm cursion_server cursion_beat cursion_celery && + docker compose -f docker-compose.yml up --build +} || { + docker volume rm cursion_server cursion_beat cursion_celery && + docker compose -f docker-compose.yml up --build +} || { + docker compose -f docker-compose.yml up --build +} + + + +# cmd to run +# > source ./setup/scripts/local.sh \ No newline at end of file diff --git a/setup/scripts/nat.sh b/setup/scripts/nat.sh new file mode 100644 index 00000000..e0194e2e --- /dev/null +++ b/setup/scripts/nat.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# NAT + High-Concurrency HTTP CONNECT Proxy (Squid) +# Supports VPC CIDR + listen port input + +# NAT_VPC_CIDR is the VPC where k8s is located +# https://cloud.digitalocean.com/networking/vpc/ + +set -u # Treat unset variables as errors + +# =========================== +# Positional arguments +# =========================== + +NAT_VPC_CIDR="${1:-}" +NAT_LISTEN_PORT="${2:-}" +NAT_PRIVATE_IP="${3:-}" + +if [ -z "$NAT_VPC_CIDR" ]; then + read -rp "Enter VPC CIDR block (e.g., 10.124.0.0/16): " NAT_VPC_CIDR +fi + +if [ -z "$NAT_LISTEN_PORT" ]; then + read -rp "Enter Proxy Listen Port (e.g., 8888): " NAT_LISTEN_PORT +fi + +if [ -z "$NAT_PRIVATE_IP" ]; then + read -rp "Enter NAT Private IP addres (e.g., 10.124.0.29): " NAT_PRIVATE_IP +fi + +echo "" +echo "Using NAT_VPC_CIDR: $NAT_VPC_CIDR" +echo "Using NAT_LISTEN_PORT: $NAT_LISTEN_PORT" +echo "Using NAT_PRIVATE_IP: $NAT_PRIVATE_IP" +echo "" + +# update and install deps +echo "[1/8] Updating system..." +apt update -y +apt install -y squid iptables-persistent curl conntrack + +# load conntrack +echo "[2/8] Loading conntrack kernel modules..." +modprobe nf_conntrack +modprobe xt_conntrack || true + +# adding sysctl tunning +if ! grep -q "NAT_PROXY_TUNING" /etc/sysctl.conf; then +echo "[3/8] Applying sysctl tuning..." +cat <> /etc/sysctl.conf +# === NAT_PROXY_TUNING === + +# TCP backlog & performance +net.core.somaxconn = 65535 +net.core.netdev_max_backlog = 250000 + +# Orphan sockets +net.ipv4.tcp_max_orphans = 16384 + +# NAT proxy performance +net.ipv4.ip_forward = 1 +net.ipv4.ip_local_port_range = 15000 65000 + +# Conntrack table +net.netfilter.nf_conntrack_max = 262144 + +# TIME_WAIT cleanup +net.ipv4.tcp_fin_timeout = 15 +net.ipv4.tcp_tw_reuse = 1 + +# limit SYNs calls +net.ipv4.tcp_syn_retries = 2 +net.ipv4.tcp_synack_retries = 2 +net.ipv4.tcp_max_syn_backlog = 4096 +net.ipv4.tcp_abort_on_overflow = 1 + +# File descriptors +fs.file-max = 500000 + +# === END NAT_PROXY_TUNING === +EOF +fi + +sysctl -p + +# setup Squid configs +echo "[4/8] Configuring Squid..." +mv /etc/squid/squid.conf /etc/squid/squid.conf.bak + +cat < /etc/squid/squid.conf +# ================== Squid CONNECT Proxy ================== + +shutdown_lifetime 3 seconds + +# Bind ONLY to private VPC IP (prevents public access) +http_port ${NAT_PRIVATE_IP}:${NAT_LISTEN_PORT} + +# ---- ACCESS CONTROL (ORDER MATTERS) ---- +acl vpc src ${NAT_VPC_CIDR} +acl SSL_ports port 443 +acl CONNECT method CONNECT + +http_access allow vpc CONNECT SSL_ports +http_access deny all + +# ---- HARD BACKPRESSURE ---- +acl max_clients maxconn 500 +http_access deny max_clients +connect_retries 1 + +# Timeouts to kill abandoned tunnels +request_timeout 30 seconds +connect_timeout 5 seconds +read_timeout 30 seconds +client_lifetime 5 minutes +persistent_request_timeout 30 seconds +half_closed_clients off + +# Headers / buffers +request_header_max_size 64 KB +reply_header_max_size 64 KB +client_request_buffer_max_size 96 KB +request_body_max_size 0 KB + +# Connection behavior +server_persistent_connections off +client_persistent_connections off +pipeline_prefetch 0 + +# Memory safety +cache deny all +memory_pools off +cache_mem 64 MB +maximum_object_size_in_memory 32 KB + +# FDs +max_filedescriptors 65535 +workers 1 + +access_log /var/log/squid/access.log +cache_log /var/log/squid/cache.log +EOF + +# restarts squid on OOM failure +cat < /etc/systemd/system/squid.service.d/oom.conf +[Service] +OOMScoreAdjust=-900 +Restart=on-failure +RestartSec=10 +EOF + +# adding memory swap +if ! swapon --show | grep -q /swapfile; then + fallocate -l 1G /swapfile + chmod 600 /swapfile + mkswap /swapfile + swapon /swapfile + grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab +fi + +# update squid daemon +systemctl daemon-reexec +systemctl daemon-reload + +# avoids silent failures +squid -k parse + +echo "[6/8] Setting up iptables NAT..." +iptables -t nat -F +iptables -A FORWARD -s "$NAT_VPC_CIDR" -j ACCEPT +iptables -t nat -A POSTROUTING -s "$NAT_VPC_CIDR" -o eth0 -j MASQUERADE + +# default deny/lock-down +iptables -A INPUT -i lo -j ACCEPT +iptables -P INPUT DROP +iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -A INPUT -p tcp --dport $NAT_LISTEN_PORT -s $NAT_VPC_CIDR -j ACCEPT +iptables -A INPUT -p tcp --dport 22 -j ACCEPT + +netfilter-persistent save +netfilter-persistent reload + +# flush existing abusive connections +conntrack -F + +# restart squid +echo "[7/8] Restarting Squid..." +systemctl restart squid +systemctl enable squid + +# success message +echo "[8/8] Completed!" +echo "" +echo "Squid CONNECT proxy active on port: $NAT_LISTEN_PORT" +echo "Allowed VPC range: $NAT_VPC_CIDR" +echo ""